Merge remote-tracking branch 'origin/master' into ZEPPELIN-2367

# Conflicts:
#	jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

LICENSE

@@ -255,6 +255,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (Apache 2.0) Bootstrap v3.0.2 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.0.2/LICENSE
     (Apache 2.0) Software under ./bigquery/*  was developed at Google (http://www.google.com/). Licensed under the Apache v2.0 License.
     (Apache 2.0) Roboto Font (https://github.com/google/roboto/)
+    (Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras)
 
 ========================================================================
 BSD 3-Clause licenses

alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java

@@ -23,9 +23,9 @@ import java.io.PrintStream;
 import java.io.ByteArrayOutputStream;
 import java.util.*;
 
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -166,7 +166,8 @@ public class AlluxioInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
     String lastWord = "";
     if (words.length > 0) {
@@ -176,7 +177,7 @@ public class AlluxioInterpreter extends Interpreter {
     List<InterpreterCompletion>  voices = new LinkedList<>();
     for (String command : keywords) {
       if (command.startsWith(lastWord)) {
-        voices.add(new InterpreterCompletion(command, command));
+        voices.add(new InterpreterCompletion(command, command, CompletionType.command.name()));
       }
     }
     return voices;

alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java

@@ -29,6 +29,8 @@ import java.util.Properties;
 
 import alluxio.client.WriteType;
 import alluxio.client.file.URIStatus;
+
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -78,28 +80,28 @@ public class AlluxioInterpreterTest {
   @Test
   public void testCompletion() {
     List expectedResultOne = Arrays.asList(
-      new InterpreterCompletion("cat", "cat"),
-      new InterpreterCompletion("chgrp", "chgrp"),
-      new InterpreterCompletion("chmod", "chmod"),
-      new InterpreterCompletion("chown", "chown"),
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"),
-      new InterpreterCompletion("count", "count"),
-      new InterpreterCompletion("createLineage", "createLineage"));
+      new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
+      new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
+      new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
+      new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
+      new InterpreterCompletion("count", "count", CompletionType.command.name()),
+      new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name()));
     List expectedResultTwo = Arrays.asList(
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"),
-      new InterpreterCompletion("count", "count"));
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
+      new InterpreterCompletion("count", "count", CompletionType.command.name()));
     List expectedResultThree = Arrays.asList(
-      new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
-      new InterpreterCompletion("copyToLocal", "copyToLocal"));
+      new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
+      new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()));
     List expectedResultNone = new ArrayList<>();
 
-    List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
-    List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);
-    List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0);
-    List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
-    List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0);
+    List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0, null);
+    List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0, null);
+    List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0, null);
+    List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0, null);
+    List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0, null);
 
     Assert.assertEquals(expectedResultOne, resultOne);
     Assert.assertEquals(expectedResultTwo, resultTwo);

angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java

@@ -67,7 +67,8 @@ public class AngularInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java

@@ -92,7 +92,8 @@ public class BeamInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return Collections.emptyList();
   }
 

bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java

@@ -332,7 +332,8 @@ public class BigQueryInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 }

cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java

@@ -216,7 +216,8 @@ public class CassandraInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 

cassandra/src/main/scala/org/apache/zeppelin/cassandra/InterpreterLogic.scala

@@ -30,7 +30,7 @@ import com.datastax.driver.core.exceptions.DriverException
 import com.datastax.driver.core.policies.{LoggingRetryPolicy, FallthroughRetryPolicy, DowngradingConsistencyRetryPolicy, Policies}
 import org.apache.zeppelin.cassandra.TextBlockHierarchy._
 import org.apache.zeppelin.display.AngularObjectRegistry
-import org.apache.zeppelin.display.Input.ParamOption
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption
 import org.apache.zeppelin.interpreter.InterpreterResult.Code
 import org.apache.zeppelin.interpreter.{InterpreterException, InterpreterResult, InterpreterContext}
 import org.slf4j.LoggerFactory

cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java

@@ -34,7 +34,7 @@ import com.datastax.driver.core.Statement;
 
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.display.Input.ParamOption;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.junit.Rule;

dev/create_release.sh

@@ -106,8 +106,8 @@ function make_binary_release() {
 
 git_clone
 make_source_package
-make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Pscala-${SCALA_VERSION}"
-make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
+make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
+make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
 
 # remove non release files and dirs
 rm -rf "${WORKING_DIR}/zeppelin"

dev/publish_release.sh

@@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
   DO_SNAPSHOT="yes"
 fi
 
-PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Pr"
+PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
 PROJECT_OPTIONS="-pl !zeppelin-distribution"
 NEXUS_STAGING="https://repository.apache.org/service/local/staging"
 NEXUS_PROFILE="153446d1ac37c4"

docs/_includes/themes/zeppelin/_navigation.html

@@ -117,10 +117,11 @@
                 <li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
                 <li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
                 <li role="separator" class="divider"></li>
-                <li class="title"><span><b>Helium Framework</b><span></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell (Experimental)</a></li>
-                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li>
+                <li class="title"><span><b>Helium Framework (Experimental)</b></span></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization: Basics</a></li>
+                <li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization_transformation.html">Writing Zeppelin Visualization: Transformation</a></li>
                 <li role="separator" class="divider"></li>
                 <li class="title"><span><b>Advanced</b><span></li>
                 <li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>

docs/development/writingzeppelinapplication.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Application(Experimental)"
+title: "Writing a new Application"
 description: "Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. Make your own Application in Apache Zeppelin is quite easy."
 group: development
 ---
@@ -19,7 +19,7 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Application (Experimental)
+# Writing a new Application
 
 <div id="toc"></div>
 
@@ -91,19 +91,18 @@ In the Zeppelin notebook, run `%dev run` will connect to application running in
 Package file is a json file that provides information about the application.
 Json file contains the following information
 
-```
+```json
 {
-  name : "[organization].[name]",
-  description : "Description",
-  artifact : "groupId:artifactId:version",
-  className : "your.package.name.YourApplicationClass",
-  resources : [
+  "name" : "[organization].[name]",
+  "description" : "Description",
+  "artifact" : "groupId:artifactId:version",
+  "className" : "your.package.name.YourApplicationClass",
+  "resources" : [
     ["resource.name", ":resource.class.name"],
     ["alternative.resource.name", ":alternative.class.name"]
   ],
-  icon : "<i class="icon"></i>"
+  "icon" : "<i class='icon'></i>"
 }
-
 ```
 
 #### name
@@ -184,4 +183,3 @@ e.g.
 ```
 icon: "<i class='fa fa-clock-o'></i>"
 ```
-

docs/development/writingzeppelinspell.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Spell(Experimental)"
+title: "Writing a new Spell"
 description: "Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter. "
 group: development
 ---
@@ -19,16 +19,16 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Spell (Experimental)
+# Writing a new Spell
 
 <div id="toc"></div>
 
-## What is Apache Zeppelin Spell 
+## What is Apache Zeppelin Spell
 
-Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter. 
+Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter.
 It can provide many benefits.
 
-- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry. 
+- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry.
 - Every spell is written in javascript. It means you can use existing javascript libraries whatever you want.
 - Spell runs on browser like display system (`%html`, `%table`). In other words, every spell can be used as display system as well.
 
@@ -57,7 +57,7 @@ For example, Use `%echo` for the Echo Spell.
 <img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_spell_using.gif" />
 
 
-## Write a new Spell 
+## Write a new Spell
 
 Making a new spell is similar to [Helium Visualization#write-new-visualization](./writingzeppelinvisualization.html#write-new-visualization).
 
@@ -67,14 +67,14 @@ Making a new spell is similar to [Helium Visualization#write-new-visualization](
 
 ### 1. Create a npm package
 
-Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell. 
+Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell.
 
 - You have to add a framework called `zeppelin-spell` as a dependency to create spell ([zeppelin-spell](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/spell))
 - Also, you can add any dependencies you want to utilise.
 
 Here's an example
 
-```
+```json
 {
   "name": "zeppelin-echo-spell",
   "description": "Zeppelin Echo Spell (example)",
@@ -95,7 +95,7 @@ Here's an example
 }
 ```
 
-### 2. Write spell using framework 
+### 2. Write spell using framework
 
 Here are some examples you can refer
 
@@ -106,7 +106,7 @@ Here are some examples you can refer
 
 Now, you need to write code to create spell which processing text.
 
-```javascript
+```js
 import {
     SpellBase,
     SpellResult,
@@ -121,8 +121,8 @@ export default class EchoSpell extends SpellBase {
 
     interpret(paragraphText) {
         const processed = paragraphText + '!';
-        
-        /** 
+
+        /**
          * should return `SpellResult` which including `data` and `type`
          * default type is `TEXT` if you don't specify.  
          */
@@ -133,7 +133,7 @@ export default class EchoSpell extends SpellBase {
 
 Here is another example. Let's say we want to create markdown spell. First of all, we should add a dependency for markdown in package.json
 
-```
+```json
 // package.json
  "dependencies": {
     "markdown": "0.5.0",
@@ -143,7 +143,7 @@ Here is another example. Let's say we want to create markdown spell. First of al
 
 And here is spell code.
 
-```javascript
+```js
 import {
     SpellBase,
     SpellResult,
@@ -171,16 +171,16 @@ export default class MarkdownSpell extends SpellBase {
 }
 ```
 
-- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js) 
+- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js)
 - You might want to return promise not string (e.g API call), then refer [Google Translation API Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-translator/index.js)
 
-### 3. Create __Helium package file__ for local deployment 
+### 3. Create __Helium package file__ for local deployment
 
 You don't want to publish your package every time you make a change in your spell. Zeppelin provides local deploy.
-The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy. 
+The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy.
 It's automatically created when you publish to npm repository but in local case, you should make it by yourself.
 
-```
+```json
 {
   "type" : "SPELL",
   "name" : "zeppelin-echo-spell",
@@ -197,18 +197,18 @@ It's automatically created when you publish to npm repository but in local case,
 
 - Place this file in your local registry directory (default `$ZEPPELIN_HOME/helium`).
 - `type` should be `SPELL`
-- Make sure that `artifact` should be same as your spell directory. 
+- Make sure that `artifact` should be same as your spell directory.
 - You can get information about other fields in [Helium Visualization#3-create-helium-package-file-and-locally-deploy](./writingzeppelinvisualization.html#3-create-helium-package-file-and-locally-deploy).
 
 ### 4. Run in dev mode
 
-```
+```bash
 cd zeppelin-web
-yarn run dev:helium 
+yarn run dev:helium
 ```
 
 You can browse localhost:9000. Every time refresh your browser, Zeppelin will rebuild your spell and reload changes.
 
-### 5. Publish your spell to the npm repository 
+### 5. Publish your spell to the npm repository
 
 See [Publishing npm packages](https://docs.npmjs.com/getting-started/publishing-npm-packages)

docs/development/writingzeppelinvisualization.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: "Writing a new Visualization(Experimental)"
+title: "Writing a new Visualization"
 description: "Apache Zeppelin Visualization is a pluggable package that can be loaded/unloaded on runtime through Helium framework in Zeppelin. A Visualization is a javascript npm package and user can use them just like any other built-in visualization in a note."
 group: development
 ---
@@ -19,7 +19,7 @@ limitations under the License.
 -->
 {% include JB/setup %}
 
-# Writing a new Visualization (Experimental)
+# Writing a new Visualization
 
 <div id="toc"></div>
 
@@ -61,11 +61,11 @@ User can use just like any other built-in visualizations.
 
 #### 1. Create a npm package
 
-Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. Normally, you can add any dependencies in package.json however Zeppelin Visualization package only allows two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).
+Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. You can add any dependencies in package.json, but you **must include two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).**
 
 Here's an example
 
-```
+```json
 {
   "name": "zeppelin_horizontalbar",
   "description" : "Horizontal Bar chart",
@@ -86,7 +86,7 @@ To create your own visualization, you need to create a js file and import [Visua
 
 [Visualization](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/visualization/visualization.js) class, there're several methods that you need to override and implement. Here's simple visualization that just prints `Hello world`.
 
-```
+```js
 import Visualization from 'zeppelin-vis'
 import PassthroughTransformation from 'zeppelin-tabledata/passthrough'
 
@@ -118,7 +118,7 @@ Zeppelin's built-in visualization uses the same API, so you can check [built-in
 __Helium Package file__ is a json file that provides information about the application.
 Json file contains the following information
 
-```
+```json
 {
   "type" : "VISUALIZATION",
   "name" : "zeppelin_horizontalbar",
@@ -154,15 +154,15 @@ e.g.
 
 When artifact exists in npm repository
 
-```
-artifact: "my-visualiztion@1.0.0"
+```json
+"artifact": "my-visualiztion@1.0.0"
 ```
 
 
 When artifact exists in local file system
 
-```
-artifact: "/path/to/my/visualization"
+```json
+"artifact": "/path/to/my/visualization"
 ```
 
 ##### license
@@ -171,8 +171,8 @@ License information.
 
 e.g.
 
-```
-license: "Apache-2.0"
+```json
+"license": "Apache-2.0"
 ```
 
 ##### icon
@@ -181,8 +181,8 @@ Icon to be used in visualization select button. String in this field will be ren
 
 e.g.
 
-```
-icon: "<i class='fa fa-coffee'></i>"
+```json
+"icon": "<i class='fa fa-coffee'></i>"
 ```
 
 
@@ -191,9 +191,9 @@ icon: "<i class='fa fa-coffee'></i>"
 Place your __Helium package file__ in local registry (ZEPPELIN_HOME/helium).
 Run Zeppelin. And then run zeppelin-web in visualization dev mode.
 
-```
+```bash
 cd zeppelin-web
-yarn run dev:helium 
+yarn run dev:helium
 ```
 
 You can browse localhost:9000. Everytime refresh your browser, Zeppelin will rebuild your visualization and reload changes.
@@ -202,4 +202,4 @@ You can browse localhost:9000. Everytime refresh your browser, Zeppelin will reb
 #### 5. Publish your visualization
 
 Once it's done, publish your visualization package using `npm publish`.
-That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.
+That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.

docs/development/writingzeppelinvisualization_transformation.md

@@ -0,0 +1,281 @@
+---
+layout: page
+title: "Transformations for Zeppelin Visualization"
+description: "Description for Transformations"
+group: development
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Transformations for Zeppelin Visualization
+
+<div id="toc"></div>
+
+## Overview 
+
+Transformations 
+
+- **renders** setting which allows users to set columns and 
+- **transforms** table rows according to the configured columns.
+
+Zeppelin provides 4 types of transformations.
+
+## 1. PassthroughTransformation
+
+`PassthroughTransformation` is the simple transformation which does not convert original tabledata at all.
+
+See [passthrough.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/passthrough.js)
+
+## 2. ColumnselectorTransformation
+
+`ColumnselectorTransformation` is uses when you need `N` axes but do not need aggregation. 
+
+See [columnselector.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/columnselector.js)
+
+## 3. PivotTransformation
+
+`PivotTransformation` provides group by and aggregation. Every chart using `PivotTransformation` has 3 axes. `Keys`, `Groups` and `Values`.
+
+See [pivot.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/pivot.js)
+
+## 4. AdvancedTransformation
+
+`AdvancedTransformation` has more detailed options while providing existing features of `PivotTransformation` and `ColumnselectorTransformation`
+
+- multiple sub charts
+- configurable chart axes
+- parameter widgets: `input`, `checkbox`, `option`, `textarea`
+- parsing parameters automatically based on their types
+- expand / fold axis and parameter panels
+- multiple transformation methods while supporting lazy converting 
+- re-initialize the whole configuration based on spec hash.
+
+### Spec 
+
+`AdvancedTransformation` requires `spec` which includes axis and parameter details for charts.
+
+Let's create 2 sub-charts called `line` and `no-group`. Each sub chart can have different axis and parameter depending on their requirements.
+
+<br/>
+
+```js
+class AwesomeVisualization extends Visualization {
+  constructor(targetEl, config) {
+    super(targetEl, config)
+  
+    const spec = {
+      charts: {
+        'line': {
+          transform: { method: 'object', },
+          sharedAxis: false, /** set if you want to share axes between sub charts, default is `false` */
+          axis: {
+            'xAxis': { dimension: 'multiple', axisType: 'key', description: 'serial', },
+            'yAxis': { dimension: 'multiple', axisType: 'aggregator', description: 'serial', },
+            'category': { dimension: 'multiple', axisType: 'group', description: 'categorical', },
+          },
+          parameter: {
+            'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+            'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
+            'lineWidth': { valueType: 'int', defaultValue: 0, description: 'width of line', },
+          },
+        },
+  
+        'no-group': {
+          transform: { method: 'object', },
+          sharedAxis: false,
+          axis: {
+            'xAxis': { dimension: 'single', axisType: 'key', },
+            'yAxis': { dimension: 'multiple', axisType: 'value', },
+          },
+          parameter: {
+            'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+            'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
+        },
+      },
+    }
+
+    this.transformation = new AdvancedTransformation(config, spec)
+  }
+  
+  ...
+  
+  // `render` will be called whenever `axis` or `parameter` is changed 
+  render(data) {
+    const { chart, parameter, column, transformer, } = data
+   
+    if (chart === 'line') {
+      const transformed = transformer()
+      // draw line chart 
+    } else if (chart === 'no-group') {
+      const transformed = transformer()
+      // draw no-group chart 
+    }
+  }
+}
+```
+
+<br/>
+
+### Spec: `axis` 
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`dimension` | `single` | Axis can contains only 1 column |
+|`dimension` | `multiple` | Axis can contains multiple columns |
+|`axisType` | `key` | Column(s) in this axis will be used as `key` like in `PivotTransformation`. These columns will be served in `column.key` |
+|`axisType` | `aggregator` | Column(s) in this axis will be used as `value` like in `PivotTransformation`. These columns will be served in `column.aggregator` |
+|`axisType` | `group` | Column(s) in this axis will be used as `group` like in `PivotTransformation`. These columns will be served in `column.group` |
+|`axisType` | (string) | Any string value can be used here. These columns will be served in `column.custom` |
+|`maxAxisCount` (optional) | (int) | The max number of columns that this axis can contain. (unlimited if `undefined`) |
+|`minAxisCount` (optional) | (int) | The min number of columns that this axis should contain to draw chart. (`1` in case of single dimension) |
+|`description` (optional) | (string) | Description for the axis. |
+
+<br/>
+
+Here is an example.
+
+```js
+axis: {
+  'xAxis': { dimension: 'multiple', axisType: 'key',  },
+  'yAxis': { dimension: 'multiple', axisType: 'aggregator'},
+  'category': { dimension: 'multiple', axisType: 'group', maxAxisCount: 2, valueType: 'string', },
+},
+```
+
+<br/>
+
+### Spec: `sharedAxis` 
+
+If you set `sharedAxis: false` for sub charts, then their axes are persisted in global space (shared). It's useful for when you creating multiple sub charts sharing their axes but have different parameters. For example, 
+
+- `basic-column`, `stacked-column`, `percent-column`
+- `pie` and `donut`
+
+<br/>
+
+Here is an example.
+
+```js
+    const spec = {
+      charts: {
+        'column': {
+          transform: { method: 'array', },
+          sharedAxis: true,
+          axis: { ... },
+          parameter: { ... },
+        },
+
+        'stacked': {
+          transform: { method: 'array', },
+          sharedAxis: true,
+          axis: { ... }
+          parameter: { ... },
+        },
+```
+
+<br/>
+
+### Spec: `parameter` 
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`valueType` | `string` | Parameter which has string value |
+|`valueType` | `int` | Parameter which has int value |
+|`valueType` | `float` | Parameter which has float value |
+|`valueType` | `boolean` | Parameter which has boolean value used with `checkbox` widget usually |
+|`valueType` | `JSON` | Parameter which has JSON value used with `textarea` widget usually. `defaultValue` should be `""` (empty string). This ||`defaultValue` | (any) | Default value of this parameter. `JSON` type should have `""` (empty string) |
+|`description` | (string) | Description of this parameter. This value will be parsed as HTML for pretty output |
+|`widget` | `input` |  Use [input](https://developer.mozilla.org/en/docs/Web/HTML/Element/input) widget. This is the default widget (if `widget` is undefined)|
+|`widget` | `checkbox` |  Use [checkbox](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/checkbox) widget. |
+|`widget` | `textarea` |  Use [textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) widget. |
+|`widget` | `option` |  Use [select + option](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select) widget. This parameter should have `optionValues` field as well. |
+|`optionValues` | (Array<string>) |  Available option values used with the `option` widget |
+
+<br/>
+
+Here is an example.
+
+```js
+parameter: {
+  // string type, input widget
+  'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
+
+  // boolean type, checkbox widget
+  'inverted': { widget: 'checkbox', valueType: 'boolean', defaultValue: false, description: 'invert x and y axes', },
+
+  // string type, option widget with `optionValues`
+  'graphType': { widget: 'option', valueType: 'string', defaultValue: 'line', description: 'graph type', optionValues: [ 'line', 'smoothedLine', 'step', ], },
+
+  // HTML in `description`
+  'dateFormat': { valueType: 'string', defaultValue: '', description: 'format of date (<a href="https://docs.amcharts.com/3/javascriptcharts/AmGraph#dateFormat">doc</a>) (e.g YYYY-MM-DD)', },
+ 
+  // JSON type, textarea widget
+  'yAxisGuides': { widget: 'textarea', valueType: 'JSON', defaultValue: '', description: 'guides of yAxis ', },
+```
+
+<br/>
+
+### Spec: `transform`
+
+| Field Name | Available Values (type) | Description |
+| --- | --- | --- |
+|`method` | `object` |  designed for rows requiring object manipulation | 
+|`method` | `array` |  designed for rows requiring array manipulation | 
+|`method` | `array:2-key` |  designed for xyz charts (e.g bubble chart) | 
+|`method` | `drill-down` |  designed for drill-down charts | 
+|`method` | `raw` | will return the original `tableData.rows` | 
+
+<br/>
+
+Whatever you specified as `transform.method`, the `transformer` value will be always function for lazy computation. 
+
+```js
+// advanced-transformation.util#getTransformer
+
+if (transformSpec.method === 'raw') {
+  transformer = () => { return rows; }
+} else if (transformSpec.method === 'array') {
+  transformer = () => {
+    ...
+    return { ... }
+  }
+}
+```
+
+Here is actual usage.
+
+```js
+class AwesomeVisualization extends Visualization {
+  constructor(...) { /** setup your spec */ }
+  
+  ... 
+  
+  // `render` will be called whenever `axis` or `parameter` are changed
+  render(data) {
+    const { chart, parameter, column, transformer, } = data
+   
+    if (chart === 'line') {
+      const transformed = transformer()
+      // draw line chart 
+    } else if (chart === 'no-group') {
+      const transformed = transformer()
+      // draw no-group chart 
+    }
+  }
+  
+  ...
+}
+```
+

docs/index.md

@@ -172,10 +172,11 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
   * [Shiro Authentication](./security/shiroauthentication.html)
   * [Notebook Authorization](./security/notebook_authorization.html)
   * [Data Source Authorization](./security/datasource_authorization.html)
-* Helium Framework
-  * [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
-  * [Writing Zeppelin Spell (Experimental)](./development/writingzeppelinspell.html)
-  * [Writing Zeppelin Visualization (Experimental)](./development/writingzeppelinvisualization.html)
+* Helium Framework (Experimental)
+  * [Writing Zeppelin Application](./development/writingzeppelinapplication.html)
+  * [Writing Zeppelin Spell](./development/writingzeppelinspell.html)
+  * [Writing Zeppelin Visualization: Basic](./development/writingzeppelinvisualization.html)
+  * [Writing Zeppelin Visualization: Transformation](./development/writingzeppelinvisualization_transformation.html)
 * Advanced
   * [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
   * [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)

docs/install/build.md

@@ -69,7 +69,7 @@ If you're unsure about the options, use the same commands that creates official
 # update all pom.xml to use scala 2.11
 ./dev/change_scala_version.sh 2.11
 # build zeppelin with all interpreters and include latest version of Apache spark support for local mode.
-mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Pr -Pscala-2.11
+mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pr -Pscala-2.11
 ```
 
 ####3. Done
@@ -140,11 +140,6 @@ Available profiles are
 -Pscala-2.11
 ```
 
-##### `-Pyarn` (optional)
-
-enable YARN support for local mode
-> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead.
-
 ##### `-Pr` (optional)
 
 enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration.
@@ -180,14 +175,14 @@ Here are some examples with several options:
 ```bash
 # build with spark-2.1, scala-2.11
 ./dev/change_scala_version.sh 2.11
-mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Pscala-2.11 -DskipTests
+mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pscala-2.11 -DskipTests
 
 # build with spark-2.0, scala-2.11
 ./dev/change_scala_version.sh 2.11
-mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Pscala-2.11 -DskipTests
+mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pscala-2.11 -DskipTests
 
 # build with spark-1.6, scala-2.10
-mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -DskipTests
+mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
 
 # spark-cassandra integration
 mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests -DskipTests
@@ -320,10 +315,10 @@ mvn clean package -Pbuild-distr
 To build a distribution with specific profiles, run:
 
 ```sh
-mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn
+mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4
 ```
 
-The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.  
+The profiles `-Pspark-1.5 -Phadoop-2.4` can be adjusted if you wish to build to a specific spark versions.  
 
 The archive is generated under _`zeppelin-distribution/target`_ directory
 

docs/install/upgrade.md

@@ -60,4 +60,4 @@ So, copying `notebook` and `conf` directory should be enough.
 ### Upgrading from Zeppelin 0.7 to 0.8
 
  - From 0.8, we recommend to use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` instead of `zeppelin.pyspark.python` as `zeppelin.pyspark.python` only effects driver. You can use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` as using them in spark.
- 
+ - From 0.8, depending on your device, the keyboard shortcut `Ctrl-L` or `Command-L` which goes to the line somewhere user wants is not supported. 

docs/interpreter/jdbc.md

@@ -123,6 +123,11 @@ The JDBC interpreter properties are defined by default like below.
     <td></td>
     <td>Some SQL which executes while opening connection</td>
   </tr>
+  <tr>
+    <td>default.completer.schemaFilters</td>
+    <td></td>
+    <td>Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)</td>
+  </tr>
 </table>
 
 If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.

elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java

@@ -33,6 +33,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.elasticsearch.action.ActionResponse;
 import org.apache.zeppelin.elasticsearch.action.AggWrapper;
 import org.apache.zeppelin.elasticsearch.action.HitWrapper;
@@ -239,12 +240,13 @@ public class ElasticsearchInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
     final List suggestions = new ArrayList<>();
 
     for (final String cmd : COMMANDS) {
       if (cmd.toLowerCase().contains(s)) {
-        suggestions.add(new InterpreterCompletion(cmd, cmd));
+        suggestions.add(new InterpreterCompletion(cmd, cmd, CompletionType.command.name()));
       }
     }
     return suggestions;

elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java

@@ -31,6 +31,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.lang.math.RandomUtils;
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -305,12 +306,12 @@ public class ElasticsearchInterpreterTest {
 
   @Theory
   public void testCompletion(ElasticsearchInterpreter interpreter) {
-    final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count"));
-    final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help"));
+    final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count", CompletionType.command.name()));
+    final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help", CompletionType.command.name()));
 
-    final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0);
-    final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0);
-    final List<InterpreterCompletion> resultAll = interpreter.completion("", 0);
+    final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0, null);
+    final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0, null);
+    final List<InterpreterCompletion> resultAll = interpreter.completion("", 0, null);
 
     Assert.assertEquals(expectedResultOne, resultOne);
     Assert.assertEquals(expectedResultTwo, resultTwo);

file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java

@@ -166,7 +166,8 @@ public abstract class FileInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java

@@ -23,6 +23,8 @@ import java.util.*;
 
 import com.google.gson.Gson;
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 
@@ -247,21 +249,25 @@ public class HDFSFileInterpreter extends FileInterpreter {
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     logger.info("Completion request at position\t" + cursor + " in string " + buf);
     final List<InterpreterCompletion> suggestions = new ArrayList<>();
     if (StringUtils.isEmpty(buf)) {
-      suggestions.add(new InterpreterCompletion("ls", "ls"));
-      suggestions.add(new InterpreterCompletion("cd", "cd"));
-      suggestions.add(new InterpreterCompletion("pwd", "pwd"));
+      suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
+      suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name()));
+      suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
       return suggestions;
     }
 
     //part of a command == no spaces
     if (buf.split(" ").length == 1){
-      if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd"));
-      if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls"));
-      if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd"));
+      if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd",
+          CompletionType.command.name()));
+      if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls",
+          CompletionType.command.name()));
+      if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd",
+          CompletionType.command.name()));
 
       return suggestions;
     }
@@ -298,7 +304,8 @@ public class HDFSFileInterpreter extends FileInterpreter {
                 String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1);
                 //beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
                 String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length());
-                suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish));
+                suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish,
+                    CompletionType.path.name()));
               }
             }
             return suggestions;

file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java

@@ -21,6 +21,8 @@ package org.apache.zeppelin.file;
 import com.google.gson.Gson;
 import junit.framework.TestCase;
 import static org.junit.Assert.*;
+
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.junit.Test;
@@ -106,11 +108,11 @@ public class HDFSFileInterpreterTest extends TestCase {
 
       // auto completion test
       List expectedResultOne = Arrays.asList(
-        new InterpreterCompletion("ls", "ls"));
+        new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
       List expectedResultTwo = Arrays.asList(
-        new InterpreterCompletion("pwd", "pwd"));
-      List<InterpreterCompletion> resultOne = t.completion("l", 0);
-      List<InterpreterCompletion> resultTwo = t.completion("p", 0);
+        new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
+      List<InterpreterCompletion> resultOne = t.completion("l", 0, null);
+      List<InterpreterCompletion> resultTwo = t.completion("p", 0, null);
 
       assertEquals(expectedResultOne, resultOne);
       assertEquals(expectedResultTwo, resultTwo);

flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java

@@ -373,7 +373,8 @@ public class FlinkInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java

@@ -282,7 +282,8 @@ public class GeodeOqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java

@@ -36,7 +36,7 @@ import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.display.Input.ParamOption;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
 import org.apache.zeppelin.annotation.ZeppelinApi;
 import org.apache.zeppelin.interpreter.RemoteWorksController;
 import org.apache.zeppelin.interpreter.InterpreterException;

groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java

@@ -121,7 +121,8 @@ public class GroovyInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+                                                InterpreterContext interpreterContext) {
     return null;
   }
 

hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java

@@ -145,7 +145,8 @@ public class HbaseInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java

@@ -98,7 +98,8 @@ public class DevInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java

@@ -331,7 +331,8 @@ public class IgniteInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java

@@ -184,7 +184,8 @@ public class IgniteSqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new LinkedList<>();
   }
 }

jdbc/pom.xml

@@ -35,7 +35,6 @@
   <properties>
     <!--library versions-->
     <postgresql.version>9.4-1201-jdbc41</postgresql.version>
-    <jline.version>2.12.1</jline.version>
     <hadoop.common.version>2.7.2</hadoop.common.version>
     <h2.version>1.4.190</h2.version>
     <commons.dbcp2.version>2.0.1</commons.dbcp2.version>
@@ -68,17 +67,6 @@
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
 	
-	<dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>jline</groupId>
-      <artifactId>jline</artifactId>
-      <version>${jline.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>com.h2database</groupId>
       <artifactId>h2</artifactId>

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

@@ -57,9 +57,7 @@ import org.apache.zeppelin.user.UsernamePassword;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
 import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
 
 import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
 import static org.apache.commons.lang.StringUtils.isEmpty;
@@ -103,6 +101,7 @@ public class JDBCInterpreter extends Interpreter {
   static final String USER_KEY = "user";
   static final String PASSWORD_KEY = "password";
   static final String PRECODE_KEY = "precode";
+  static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters";
   static final String JDBC_JCEKS_FILE = "jceks.file";
   static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
   static final String PRECODE_KEY_TEMPLATE = "%s.precode";
@@ -130,22 +129,12 @@ public class JDBCInterpreter extends Interpreter {
 
   private final HashMap<String, Properties> basePropretiesMap;
   private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
-  private final Map<String, SqlCompleter> propertyKeySqlCompleterMap;
 
-  private static final Function<CharSequence, InterpreterCompletion> sequenceToStringTransformer =
-      new Function<CharSequence, InterpreterCompletion>() {
-        public InterpreterCompletion apply(CharSequence seq) {
-          return new InterpreterCompletion(seq.toString(), seq.toString());
-        }
-      };
-
-  private static final List<InterpreterCompletion> NO_COMPLETION = new ArrayList<>();
   private int maxLineResults;
 
   public JDBCInterpreter(Properties property) {
     super(property);
     jdbcUserConfigurationsMap = new HashMap<>();
-    propertyKeySqlCompleterMap = new HashMap<>();
     basePropretiesMap = new HashMap<>();
     maxLineResults = MAX_LINE_DEFAULT;
   }
@@ -190,9 +179,6 @@ public class JDBCInterpreter extends Interpreter {
     }
     logger.debug("JDBC PropretiesMap: {}", basePropretiesMap);
 
-    for (String propertyKey : basePropretiesMap.keySet()) {
-      propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
-    }
     setMaxLineResults();
   }
 
@@ -203,10 +189,11 @@ public class JDBCInterpreter extends Interpreter {
     }
   }
 
-  private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
-
+  private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) {
+    String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
+    String filters = getProperty(schemaFiltersKey);
     SqlCompleter completer = new SqlCompleter();
-    completer.initFromConnection(jdbcConnection, "");
+    completer.initFromConnection(jdbcConnection, filters);
     return completer;
   }
 
@@ -418,7 +405,7 @@ public class JDBCInterpreter extends Interpreter {
                 user, propertyKey, properties);
       }
     }
-    propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
+
     return connection;
   }
 
@@ -802,18 +789,26 @@ public class JDBCInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
-    List<CharSequence> candidates = new ArrayList<>();
-    SqlCompleter sqlCompleter = propertyKeySqlCompleterMap.get(getPropertyKey(buf));
-    // It's strange but here cursor comes with additional +1 (even if buf is "" cursor = 1)
-    if (sqlCompleter != null && sqlCompleter.complete(buf, cursor - 1, candidates) >= 0) {
-      List<InterpreterCompletion> completion;
-      completion = Lists.transform(candidates, sequenceToStringTransformer);
-
-      return completion;
-    } else {
-      return NO_COMPLETION;
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
+    List<InterpreterCompletion> candidates = new ArrayList<>();
+    String propertyKey = getPropertyKey(buf);
+    Connection connection = null;
+    try {
+      if (interpreterContext != null) {
+        connection = getConnection(propertyKey, interpreterContext);
+      }
+    } catch (ClassNotFoundException | SQLException | IOException e) {
+      logger.warn("SQLCompleter will created without use connection");
     }
+
+    SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey);
+
+    if (sqlCompleter != null) {
+      sqlCompleter.complete(buf, cursor - 1, candidates);
+    }
+
+    return candidates;
   }
 
   public int getMaxResult() {

jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java

@@ -4,15 +4,6 @@ package org.apache.zeppelin.jdbc;
  * This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE
  */
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Sets;
-import com.google.common.collect.Sets.SetView;
-import jline.console.completer.ArgumentCompleter.ArgumentList;
-import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
-import jline.console.completer.StringsCompleter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -20,15 +11,34 @@ import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.TreeSet;
 import java.util.regex.Pattern;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.completer.StringsCompleter;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import jline.console.completer.ArgumentCompleter.ArgumentList;
+import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
+
 import static org.apache.commons.lang.StringUtils.isBlank;
 
 /**
  * SQL auto complete functionality for the JdbcInterpreter.
  */
-public class SqlCompleter extends StringsCompleter {
+public class SqlCompleter {
 
   private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
 
@@ -67,8 +77,7 @@ public class SqlCompleter extends StringsCompleter {
    */
   private StringsCompleter keywordCompleter = new StringsCompleter();
 
-  @Override
-  public int complete(String buffer, int cursor, List<CharSequence> candidates) {
+  public int complete(String buffer, int cursor, List<InterpreterCompletion> candidates) {
 
     logger.debug("Complete with buffer = " + buffer + ", cursor = " + cursor);
 
@@ -76,21 +85,36 @@ public class SqlCompleter extends StringsCompleter {
     // white spaces.
     ArgumentList argumentList = sqlDelimiter.delimit(buffer, cursor);
 
-    String beforeCursorBuffer = buffer.substring(0,
-            Math.min(cursor, buffer.length())).toUpperCase();
+    Pattern whitespaceEndPatter = Pattern.compile("\\s$");
+    String cursorArgument = null;
+    int argumentPosition;
+    if (buffer.length() == 0 || whitespaceEndPatter.matcher(buffer).find()) {
+      argumentPosition = buffer.length() - 1;
+    } else {
+      cursorArgument = argumentList.getCursorArgument();
+      argumentPosition = argumentList.getArgumentPosition();
+    }
 
-    // check what sql is and where cursor is to allow column completion or not
     boolean isColumnAllowed = true;
-    if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
-            && !beforeCursorBuffer.contains(" WHERE "))
-      isColumnAllowed = false;
+    if (buffer.length() > 0) {
+      String beforeCursorBuffer = buffer.substring(0,
+          Math.min(cursor, buffer.length())).toUpperCase();
+      // check what sql is and where cursor is to allow column completion or not
+      if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
+          && !beforeCursorBuffer.contains(" WHERE "))
+        isColumnAllowed = false;
+    }
 
-    int complete = completeName(argumentList.getCursorArgument(),
-            argumentList.getArgumentPosition(), candidates,
+    int complete = completeName(cursorArgument, argumentPosition, candidates,
             findAliasesInSQL(argumentList.getArguments()), isColumnAllowed);
 
+    if (candidates.size() == 1) {
+      InterpreterCompletion interpreterCompletion = candidates.get(0);
+      interpreterCompletion.setName(interpreterCompletion.getName() + " ");
+      interpreterCompletion.setValue(interpreterCompletion.getValue() + " ");
+      candidates.set(0, interpreterCompletion);
+    }
     logger.debug("complete:" + complete + ", size:" + candidates.size());
-
     return complete;
   }
 
@@ -98,24 +122,26 @@ public class SqlCompleter extends StringsCompleter {
    * Return list of schema names within the database
    *
    * @param meta metadata from connection to database
-   * @param schemaFilter a schema name pattern; must match the schema name
+   * @param schemaFilters a schema name patterns; must match the schema name
    *        as it is stored in the database; "" retrieves those without a schema;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @return set of all schema names in the database
    */
-  private static Set<String> getSchemaNames(DatabaseMetaData meta, String schemaFilter) {
+  private static Set<String> getSchemaNames(DatabaseMetaData meta, List<String> schemaFilters) {
     Set<String> res = new HashSet<>();
     try {
       ResultSet schemas = meta.getSchemas();
       try {
         while (schemas.next()) {
           String schemaName = schemas.getString("TABLE_SCHEM");
-          if (schemaName == null)
+          if (schemaName == null) {
             schemaName = "";
-          if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
-                  schemaFilter.replace("_", ".").replace("%", ".*?"))) {
-            res.add(schemaName);
+          }
+          for (String schemaFilter : schemaFilters) {
+            if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
+              res.add(schemaName);
+            }
           }
         }
       } finally {
@@ -131,22 +157,23 @@ public class SqlCompleter extends StringsCompleter {
    * Return list of catalog names within the database
    *
    * @param meta metadata from connection to database
-   * @param schemaFilter a catalog name pattern; must match the catalog name
+   * @param schemaFilters a catalog name patterns; must match the catalog name
    *        as it is stored in the database; "" retrieves those without a catalog;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @return set of all catalog names in the database
    */
-  private static Set<String> getCatalogNames(DatabaseMetaData meta, String schemaFilter) {
+  private static Set<String> getCatalogNames(DatabaseMetaData meta, List<String> schemaFilters) {
     Set<String> res = new HashSet<>();
     try {
       ResultSet schemas = meta.getCatalogs();
       try {
         while (schemas.next()) {
           String schemaName = schemas.getString("TABLE_CAT");
-          if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
-                  schemaFilter.replace("_", ".").replace("%", ".*?"))) {
-            res.add(schemaName);
+          for (String schemaFilter : schemaFilters) {
+            if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
+              res.add(schemaName);
+            }
           }
         }
       } finally {
@@ -166,7 +193,7 @@ public class SqlCompleter extends StringsCompleter {
    * @param schemaFilter a schema name pattern; must match the schema name
    *        as it is stored in the database; "" retrieves those without a schema;
    *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   *        the search; supports '%'; for example "prod_v_%"
    * @param tables function fills this map, for every schema name adds
    *        set of table names within the schema
    * @param columns function fills this map, for every table name adds set
@@ -177,19 +204,27 @@ public class SqlCompleter extends StringsCompleter {
                                               Map<String, Set<String>> tables,
                                               Map<String, Set<String>> columns)  {
     try {
-      ResultSet cols = meta.getColumns(catalogName, schemaFilter, "%",
-              "%");
+      ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%");
       try {
         while (cols.next()) {
           String schema = cols.getString("TABLE_SCHEM");
-          if (schema == null) schema = cols.getString("TABLE_CAT");
+          if (schema == null) {
+            schema = cols.getString("TABLE_CAT");
+          }
+          if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) {
+            continue;
+          }
           String table = cols.getString("TABLE_NAME");
           String column = cols.getString("COLUMN_NAME");
           if (!isBlank(table)) {
             String schemaTable = schema + "." + table;
-            if (!columns.containsKey(schemaTable)) columns.put(schemaTable, new HashSet<String>());
+            if (!columns.containsKey(schemaTable)) {
+              columns.put(schemaTable, new HashSet<String>());
+            }
             columns.get(schemaTable).add(column);
-            if (!tables.containsKey(schema)) tables.put(schema, new HashSet<String>());
+            if (!tables.containsKey(schema)) {
+              tables.put(schema, new HashSet<String>());
+            }
             tables.get(schema).add(table);
           }
         }
@@ -327,33 +362,31 @@ public class SqlCompleter extends StringsCompleter {
    * Initializes all local completers from database connection
    *
    * @param connection database connection
-   * @param schemaFilter a schema name pattern; must match the schema name
-   *        as it is stored in the database; "" retrieves those without a schema;
-   *        <code>null</code> means that the schema name should not be used to narrow
-   *        the search; supports '%' and '_' symbols; for example "prod_v_%"
+   * @param schemaFiltersString a comma separated schema name patterns; supports '%'  symbol;
+   * for example "prod_v_%,prod_t_%"
    */
-  public void initFromConnection(Connection connection, String schemaFilter) {
+  public void initFromConnection(Connection connection, String schemaFiltersString) {
+    if (schemaFiltersString == null) {
+      schemaFiltersString = StringUtils.EMPTY;
+    }
+    List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
 
-    try {
+    try (Connection c = connection) {
       Map<String, Set<String>> tables = new HashMap<>();
       Map<String, Set<String>> columns = new HashMap<>();
       Set<String> schemas = new HashSet<>();
       Set<String> catalogs = new HashSet<>();
       Set<String> keywords = getSqlKeywordsCompletions(connection);
       if (connection != null) {
-        schemas = getSchemaNames(connection.getMetaData(), schemaFilter);
-        catalogs = getCatalogNames(connection.getMetaData(), schemaFilter);
-
-        if (!"".equals(connection.getCatalog())) {
-          if (schemas.size() == 0 )
-            schemas.add(connection.getCatalog());
-          fillTableAndColumnNames(connection.getCatalog(), connection.getMetaData(), schemaFilter,
-                  tables, columns);
-        } else {
-          if (schemas.size() == 0) schemas.addAll(catalogs);
-          for (String catalog : catalogs) {
-            fillTableAndColumnNames(catalog, connection.getMetaData(), schemaFilter, tables,
-                    columns);
+        schemas = getSchemaNames(connection.getMetaData(), schemaFilters);
+        catalogs = getCatalogNames(connection.getMetaData(), schemaFilters);
+        if (schemas.size() == 0) {
+          schemas.addAll(catalogs);
+        }
+        for (String schema : schemas) {
+          for (String schemaFilter : schemaFilters) {
+            fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables,
+                columns);
           }
         }
       }
@@ -408,8 +441,18 @@ public class SqlCompleter extends StringsCompleter {
    */
   private int completeTable(String schema, String buffer, int cursor,
                             List<CharSequence> candidates) {
+    if (schema == null) {
+      int res = -1;
+      Set<CharSequence> candidatesSet = new HashSet<>();
+      for (StringsCompleter stringsCompleter : tablesCompleters.values()) {
+        int resTable = stringsCompleter.complete(buffer, cursor, candidatesSet);
+        res = Math.max(res, resTable);
+      }
+      candidates.addAll(candidatesSet);
+      return res;
+    }
     // Wrong schema
-    if (!tablesCompleters.containsKey(schema))
+    if (!tablesCompleters.containsKey(schema) && schema != null)
       return -1;
     else
       return tablesCompleters.get(schema).complete(buffer, cursor, candidates);
@@ -422,12 +465,23 @@ public class SqlCompleter extends StringsCompleter {
    */
   private int completeColumn(String schema, String table, String buffer, int cursor,
                              List<CharSequence> candidates) {
+    if (table == null && schema == null) {
+      int res = -1;
+      Set<CharSequence> candidatesSet = new HashSet<>();
+      for (StringsCompleter stringsCompleter : columnsCompleters.values()) {
+        int resColumn = stringsCompleter.complete(buffer, cursor, candidatesSet);
+        res = Math.max(res, resColumn);
+      }
+      candidates.addAll(candidatesSet);
+      return res;
+    }
     // Wrong schema or wrong table
     if (!tablesCompleters.containsKey(schema) ||
-            !columnsCompleters.containsKey(schema + "." + table))
+        !columnsCompleters.containsKey(schema + "." + table)) {
       return -1;
-    else
+    } else {
       return columnsCompleters.get(schema + "." + table).complete(buffer, cursor, candidates);
+    }
   }
 
   /**
@@ -438,32 +492,43 @@ public class SqlCompleter extends StringsCompleter {
    * @param isColumnAllowed if false the function will not search and complete columns
    * @return -1 in case of no candidates found, 0 otherwise
    */
-  public int completeName(String buffer, int cursor, List<CharSequence> candidates,
+  public int completeName(String buffer, int cursor, List<InterpreterCompletion> candidates,
                           Map<String, String> aliases, boolean isColumnAllowed) {
 
-    if (buffer == null) buffer = "";
-
-    // no need to process after first point after cursor
-    int nextPointPos = buffer.indexOf('.', cursor);
-    if (nextPointPos != -1) buffer = buffer.substring(0, nextPointPos);
-
     // points divide the name to the schema, table and column - find them
-    int pointPos1 = buffer.indexOf('.');
-    int pointPos2 = buffer.indexOf('.', pointPos1 + 1);
+    int pointPos1 = -1;
+    int pointPos2 = -1;
 
+    if (StringUtils.isNotEmpty(buffer)) {
+      if (buffer.length() > cursor) {
+        buffer = buffer.substring(0, cursor + 1);
+      }
+      pointPos1 = buffer.indexOf('.');
+      pointPos2 = buffer.indexOf('.', pointPos1 + 1);
+    }
     // find schema and table name if they are
     String schema;
     String table;
     String column;
-    if (pointPos1 == -1) {             // process only schema or keyword case
-      schema = buffer;
-      int keywordsRes = completeKeyword(buffer, cursor, candidates);
+
+    if (pointPos1 == -1) {             // process all
+      List<CharSequence> keywordsCandidates = new ArrayList();
       List<CharSequence> schemaCandidates = new ArrayList<>();
-      int schemaRes = completeSchema(schema, cursor, schemaCandidates);
-      candidates.addAll(schemaCandidates);
-      return Math.max(keywordsRes, schemaRes);
-    }
-    else {
+      List<CharSequence> tableCandidates = new ArrayList<>();
+      List<CharSequence> columnCandidates = new ArrayList<>();
+      int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates);
+      int schemaRes = completeSchema(buffer, cursor, schemaCandidates);
+      int tableRes = completeTable(null, buffer, cursor, tableCandidates);
+      int columnRes = -1;
+      if (isColumnAllowed) {
+        columnRes = completeColumn(null, null, buffer, cursor, columnCandidates);
+      }
+      addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name());
+      addCompletions(candidates, schemaCandidates, CompletionType.schema.name());
+      addCompletions(candidates, tableCandidates, CompletionType.table.name());
+      addCompletions(candidates, columnCandidates, CompletionType.column.name());
+      return NumberUtils.max(new int[]{keywordsRes, schemaRes, tableRes, columnRes});
+    } else {
       schema = buffer.substring(0, pointPos1);
       if (aliases.containsKey(schema)) {  // process alias case
         String alias = aliases.get(schema);
@@ -471,26 +536,40 @@ public class SqlCompleter extends StringsCompleter {
         schema = alias.substring(0, pointPos);
         table = alias.substring(pointPos + 1);
         column = buffer.substring(pointPos1 + 1);
-      }
-      else if (pointPos2 == -1) {        // process schema.table case
+      } else if (pointPos2 == -1) {        // process schema.table case
+        List<CharSequence> tableCandidates = new ArrayList();
         table = buffer.substring(pointPos1 + 1);
-        return completeTable(schema, table, cursor - pointPos1 - 1, candidates);
-      }
-      else {
+        int tableRes = completeTable(schema, table, cursor - pointPos1 - 1, tableCandidates);
+        addCompletions(candidates, tableCandidates, CompletionType.table.name());
+        return tableRes;
+      } else {
         table = buffer.substring(pointPos1 + 1, pointPos2);
         column = buffer.substring(pointPos2 + 1);
       }
     }
 
     // here in case of column
-    if (isColumnAllowed)
-      return completeColumn(schema, table, column, cursor - pointPos2 - 1, candidates);
-    else
-      return -1;
+    if (table != null && isColumnAllowed) {
+      List<CharSequence> columnCandidates = new ArrayList();
+      int columnRes = completeColumn(schema, table, column, cursor - pointPos2 - 1,
+          columnCandidates);
+      addCompletions(candidates, columnCandidates, CompletionType.column.name());
+      return columnRes;
+    }
+
+    return -1;
   }
 
   // test purpose only
   WhitespaceArgumentDelimiter getSqlDelimiter() {
     return this.sqlDelimiter;
   }
+
+  private void addCompletions(List<InterpreterCompletion> interpreterCompletions,
+      List<CharSequence> candidates, String meta) {
+    for (CharSequence candidate : candidates) {
+      interpreterCompletions.add(new InterpreterCompletion(candidate.toString(),
+          candidate.toString(), meta));
+    }
+  }
 }

jdbc/src/main/resources/ansi.sql.keywords

@@ -1 +1 @@
-ABSOLUTE,ACTION,ADD,ALL,ALLOCATE,ALTER,AND,ANY,ARE,AS,ASC,ASSERTION,AT,AUTHORIZATION,AVG,BEGIN,BETWEEN,BIT,BIT_LENGTH,BOTH,BY,CASCADE,CASCADED,CASE,CAST,CATALOG,CHAR,CHARACTER,CHAR_LENGTH,CHARACTER_LENGTH,CHECK,CLOSE,CLUSTER,COALESCE,COLLATE,COLLATION,COLUMN,COMMIT,CONNECT,CONNECTION,CONSTRAINT,CONSTRAINTS,CONTINUE,CONVERT,CORRESPONDING,COUNT,CREATE,CROSS,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATE,DAY,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFERRABLE,DEFERRED,DELETE,DESC,DESCRIBE,DESCRIPTOR,DIAGNOSTICS,DISCONNECT,DISTINCT,DOMAIN,DOUBLE,DROP,ELSE,END,END-EXEC,ESCAPE,EXCEPT,EXCEPTION,EXEC,EXECUTE,EXISTS,EXTERNAL,EXTRACT,FALSE,FETCH,FIRST,FLOAT,FOR,FOREIGN,FOUND,FROM,FULL,GET,GLOBAL,GO,GOTO,GRANT,GROUP,HAVING,HOUR,IDENTITY,IMMEDIATE,IN,INDICATOR,INITIALLY,INNER,INPUT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,IS,ISOLATION,JOIN,KEY,LANGUAGE,LAST,LEADING,LEFT,LEVEL,LIKE,LOCAL,LOWER,MATCH,MAX,MIN,MINUTE,MODULE,MONTH,NAMES,NATIONAL,NATURAL,NCHAR,NEXT,NO,NOT,NULL,NULLIF,NUMERIC,OCTET_LENGTH,OF,ON,ONLY,OPEN,OPTION,OR,ORDER,OUTER,OUTPUT,OVERLAPS,OVERWRITE,PAD,PARTIAL,PARTITION,POSITION,PRECISION,PREPARE,PRESERVE,PRIMARY,PRIOR,PRIVILEGES,PROCEDURE,PUBLIC,READ,REAL,REFERENCES,RELATIVE,RESTRICT,REVOKE,RIGHT,ROLLBACK,ROWS,SCHEMA,SCROLL,SECOND,SECTION,SELECT,SESSION,SESSION_USER,SET,SIZE,SMALLINT,SOME,SPACE,SQL,SQLCODE,SQLERROR,SQLSTATE,SUBSTRING,SUM,SYSTEM_USER,TABLE,TEMPORARY,THEN,TIME,TIMESTAMP,TIMEZONE_HOUR,TIMEZONE_MINUTE,TO,TRAILING,TRANSACTION,TRANSLATE,TRANSLATION,TRIM,TRUE,UNION,UNIQUE,UNKNOWN,UPDATE,UPPER,USAGE,USER,USING,VALUE,VALUES,VARCHAR,VARYING,VIEW,WHEN,WHENEVER,WHERE,WITH,WORK,WRITE,YEAR,ZONE,ADA,C,CATALOG_NAME,CHARACTER_SET_CATALOG,CHARACTER_SET_NAME,CHARACTER_SET_SCHEMA,CLASS_ORIGIN,COBOL,COLLATION_CATALOG,COLLATION_NAME,COLLATION_SCHEMA,COLUMN_NAME,COMMAND_FUNCTION,COMMITTED,CONDITION_NUMBER,CONNECTION_NAME,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CURSOR_NAME,DATA,DATETIME_INTERVAL_CODE,DATETIME_INTERVAL_PRECISION,DYNAMIC_FUNCTION,FORTRAN,LENGTH,MESSAGE_LENGTH,MESSAGE_OCTET_LENGTH,MESSAGE_TEXT,MORE,MUMPS,NAME,NULLABLE,NUMBER,PASCAL,PLI,REPEATABLE,RETURNED_LENGTH,RETURNED_OCTET_LENGTH,RETURNED_SQLSTATE,ROW_COUNT,SCALE,SCHEMA_NAME,SERIALIZABLE,SERVER_NAME,SUBCLASS_ORIGIN,TABLE_NAME,TYPE,UNCOMMITTED,UNNAMED,LIMIT
+absolute,action,add,all,allocate,alter,and,any,are,as,asc,assertion,at,authorization,avg,begin,between,bit,bit_length,both,by,cascade,cascaded,case,cast,catalog,char,character,char_length,character_length,check,close,cluster,coalesce,collate,collation,column,commit,connect,connection,constraint,constraints,continue,convert,corresponding,count,create,cross,current,current_date,current_time,current_timestamp,current_user,cursor,date,day,deallocate,dec,decimal,declare,default,deferrable,deferred,delete,desc,describe,descriptor,diagnostics,disconnect,distinct,domain,double,drop,else,end,end-exec,escape,except,exception,exec,execute,exists,external,extract,false,fetch,first,float,for,foreign,found,from,full,get,global,go,goto,grant,group,having,hour,identity,immediate,in,indicator,initially,inner,input,insensitive,insert,int,integer,intersect,interval,into,is,isolation,join,key,language,last,leading,left,level,like,local,lower,match,max,min,minute,module,month,names,national,natural,nchar,next,no,not,null,nullif,numeric,octet_length,of,on,only,open,option,or,order,outer,output,overlaps,overwrite,pad,partial,partition,position,precision,prepare,preserve,primary,prior,privileges,procedure,public,read,real,references,relative,restrict,revoke,right,rollback,rows,schema,scroll,second,section,select,session,session_user,set,size,smallint,some,space,sql,sqlcode,sqlerror,sqlstate,substring,sum,system_user,table,temporary,then,time,timestamp,timezone_hour,timezone_minute,to,trailing,transaction,translate,translation,trim,true,union,unique,unknown,update,upper,usage,user,using,value,values,varchar,varying,view,when,whenever,where,with,work,write,year,zone,ada,c,catalog_name,character_set_catalog,character_set_name,character_set_schema,class_origin,cobol,collation_catalog,collation_name,collation_schema,column_name,command_function,committed,condition_number,connection_name,constraint_catalog,constraint_name,constraint_schema,cursor_name,data,datetime_interval_code,datetime_interval_precision,dynamic_function,fortran,length,message_length,message_octet_length,message_text,more,mumps,name,nullable,number,pascal,pli,repeatable,returned_length,returned_octet_length,returned_sqlstate,row_count,scale,schema_name,serializable,server_name,subclass_origin,table_name,type,uncommitted,unnamed,limit

jdbc/src/main/resources/interpreter-setting.json

@@ -28,6 +28,12 @@
         "defaultValue": "org.postgresql.Driver",
         "description": "JDBC Driver Name"
       },
+      "default.completer.schemaFilters": {
+        "envName": null,
+        "propertyName": "default.completer.schemaFilters",
+        "defaultValue": "",
+        "description": "Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)"
+      },
       "default.precode": {
         "envName": null,
         "propertyName": "zeppelin.jdbc.precode",

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -295,9 +296,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     jdbcInterpreter.interpret("", interpreterContext);
 
-    List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 1);
+    List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3, null);
 
-    InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ");
+    InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ", CompletionType.keyword.name());
 
     assertEquals(1, completionList.size());
     assertEquals(true, completionList.contains(correctCompletionKeyword));

jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java

@@ -14,18 +14,26 @@
  */
 package org.apache.zeppelin.jdbc;
 
-import com.google.common.base.Joiner;
-import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
-import jline.console.completer.ArgumentCompleter;
-import jline.console.completer.Completer;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.completer.CompletionType;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.*;
+import com.google.common.base.Joiner;
+import jline.console.completer.ArgumentCompleter;
 
 import static com.google.common.collect.Sets.newHashSet;
 import static org.junit.Assert.assertEquals;
@@ -34,18 +42,18 @@ import static org.junit.Assert.assertTrue;
 /**
  * SQL completer unit tests
  */
-public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
+public class SqlCompleterTest {
 
   public class CompleterTester {
 
-    private Completer completer;
+    private SqlCompleter completer;
 
     private String buffer;
     private int fromCursor;
     private int toCursor;
-    private Set<String> expectedCompletions;
+    private Set<InterpreterCompletion> expectedCompletions;
 
-    public CompleterTester(Completer completer) {
+    public CompleterTester(SqlCompleter completer) {
       this.completer = completer;
     }
 
@@ -64,7 +72,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
       return this;
     }
 
-    public CompleterTester expect(Set<String> expectedCompletions) {
+    public CompleterTester expect(Set<InterpreterCompletion> expectedCompletions) {
       this.expectedCompletions = expectedCompletions;
       return this;
     }
@@ -75,9 +83,13 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
       }
     }
 
-    private void expectedCompletions(String buffer, int cursor, Set<String> expected) {
+    private void expectedCompletions(String buffer, int cursor,
+        Set<InterpreterCompletion> expected) {
+      if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) {
+        buffer = buffer.substring(0, cursor + 1);
+      }
 
-      ArrayList<CharSequence> candidates = new ArrayList<>();
+      List<InterpreterCompletion> candidates = new ArrayList<>();
 
       completer.complete(buffer, cursor, candidates);
 
@@ -85,11 +97,15 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
 
       logger.info(explain);
 
-      assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
-              + explain, expected, newHashSet(candidates));
+      Assert.assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
+          + explain, expected, newHashSet(candidates));
     }
 
-    private String explain(String buffer, int cursor, ArrayList<CharSequence> candidates) {
+    private String explain(String buffer, int cursor, List<InterpreterCompletion> candidates) {
+      List<String> cndidateStrings = new ArrayList<>();
+      for (InterpreterCompletion candidate : candidates) {
+        cndidateStrings.add(candidate.getValue());
+      }
       StringBuffer sb = new StringBuffer();
 
       for (int i = 0; i <= Math.max(cursor, buffer.length()); i++) {
@@ -109,7 +125,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
           sb.append(")");
         }
       }
-      sb.append(" >> [").append(Joiner.on(",").join(candidates)).append("]");
+      sb.append(" >> [").append(Joiner.on(",").join(cndidateStrings)).append("]");
 
       return sb.toString();
     }
@@ -122,7 +138,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   private CompleterTester tester;
 
   private ArgumentCompleter.WhitespaceArgumentDelimiter delimiter =
-          new ArgumentCompleter.WhitespaceArgumentDelimiter();
+      new ArgumentCompleter.WhitespaceArgumentDelimiter();
 
   private SqlCompleter sqlCompleter = new SqlCompleter();
 
@@ -178,7 +194,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_Simple(){
+  public void testFindAliasesInSQL_Simple() {
     String sql = "select * from prod_emart.financial_account a";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(delimiter.delimit(sql, 0).getArguments());
     assertEquals(1, res.size());
@@ -186,7 +202,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_Two(){
+  public void testFindAliasesInSQL_Two() {
     String sql = "select * from prod_dds.financial_account a, prod_dds.customer b";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
     assertEquals(2, res.size());
@@ -195,7 +211,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testFindAliasesInSQL_WrongTables(){
+  public void testFindAliasesInSQL_WrongTables() {
     String sql = "select * from prod_ddsxx.financial_account a, prod_dds.customerxx b";
     Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
     assertEquals(0, res.size());
@@ -205,116 +221,145 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
   public void testCompleteName_Empty() {
     String buffer = "";
     int cursor = 0;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
-    sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
-    assertEquals(9, candidates.size());
-    assertTrue(candidates.contains("prod_dds"));
-    assertTrue(candidates.contains("prod_emart"));
-    assertTrue(candidates.contains("SUM"));
-    assertTrue(candidates.contains("SUBSTRING"));
-    assertTrue(candidates.contains("SUBCLASS_ORIGIN"));
-    assertTrue(candidates.contains("SELECT"));
-    assertTrue(candidates.contains("ORDER"));
-    assertTrue(candidates.contains("LIMIT"));
-    assertTrue(candidates.contains("FROM"));
+    sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
+    assertEquals(17, candidates.size());
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("customer", "customer", CompletionType.table.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("name", "name", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_SimpleSchema() {
     String buffer = "prod_";
     int cursor = 3;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("prod_dds"));
-    assertTrue(candidates.contains("prod_emart"));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
   }
 
   @Test
   public void testCompleteName_SimpleTable() {
     String buffer = "prod_dds.fin";
     int cursor = 11;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
     assertEquals(1, candidates.size());
-    assertTrue(candidates.contains("financial_account "));
+    assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
   }
 
   @Test
   public void testCompleteName_SimpleColumn() {
     String buffer = "prod_dds.financial_account.acc";
     int cursor = 30;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_WithAlias() {
     String buffer = "a.acc";
     int cursor = 4;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     aliases.put("a", "prod_dds.financial_account");
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
   @Test
   public void testCompleteName_WithAliasAndPoint() {
     String buffer = "a.";
     int cursor = 2;
-    List<CharSequence> candidates = new ArrayList<>();
+    List<InterpreterCompletion> candidates = new ArrayList<>();
     Map<String, String> aliases = new HashMap<>();
     aliases.put("a", "prod_dds.financial_account");
     sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
     assertEquals(2, candidates.size());
-    assertTrue(candidates.contains("account_rk"));
-    assertTrue(candidates.contains("account_id"));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
+    assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
   }
 
+  @Test
   public void testSchemaAndTable() {
-    String buffer = "select * from prod_v_emart.fi";
-    tester.buffer(buffer).from(15).to(26).expect(newHashSet("prod_v_emart ")).test();
-    tester.buffer(buffer).from(27).to(29).expect(newHashSet("financial_account ")).test();
+    String buffer = "select * from prod_emart.fi";
+    tester.buffer(buffer).from(19).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart ", "prod_emart ", CompletionType.schema.name()))).test();
+    tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account ", "financial_account ", CompletionType.table.name()))).test();
   }
 
   @Test
   public void testEdges() {
     String buffer = "  ORDER  ";
-    tester.buffer(buffer).from(0).to(7).expect(newHashSet("ORDER ")).test();
-    tester.buffer(buffer).from(8).to(15).expect(newHashSet("ORDER", "SUBCLASS_ORIGIN", "SUBSTRING",
-            "prod_emart", "LIMIT", "SUM", "prod_dds", "SELECT", "FROM")).test();
+    tester.buffer(buffer).from(2).to(6).expect(newHashSet(new InterpreterCompletion("ORDER ", "ORDER ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(0).to(1).expect(newHashSet(
+        new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()),
+        new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()),
+        new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
+        new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
+        new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
+        new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()),
+        new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()),
+        new InterpreterCompletion("customer", "customer", CompletionType.table.name()),
+        new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()),
+        new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()),
+        new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()),
+        new InterpreterCompletion("name", "name", CompletionType.column.name()),
+        new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()),
+        new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())
+    )).test();
   }
 
   @Test
   public void testMultipleWords() {
     String buffer = "SELE FRO LIM";
-    tester.buffer(buffer).from(0).to(4).expect(newHashSet("SELECT ")).test();
-    tester.buffer(buffer).from(5).to(8).expect(newHashSet("FROM ")).test();
-    tester.buffer(buffer).from(9).to(12).expect(newHashSet("LIMIT ")).test();
+    tester.buffer(buffer).from(1).to(3).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(6).to(7).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(9).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT ", "LIMIT ", CompletionType.keyword.name()))).test();
   }
 
   @Test
   public void testMultiLineBuffer() {
     String buffer = " \n SELE\nFRO";
-    tester.buffer(buffer).from(0).to(7).expect(newHashSet("SELECT ")).test();
-    tester.buffer(buffer).from(8).to(11).expect(newHashSet("FROM ")).test();
+    tester.buffer(buffer).from(4).to(6).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
+    tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
   }
 
   @Test
   public void testMultipleCompletionSuggestions() {
     String buffer = "SU";
-    tester.buffer(buffer).from(0).to(2).expect(newHashSet("SUBCLASS_ORIGIN", "SUM", "SUBSTRING"))
-            .test();
+    tester.buffer(buffer).from(1).to(2).expect(newHashSet(
+        new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
+        new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))
+    ).test();
   }
 
   @Test

kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java

@@ -95,7 +95,8 @@ public class KylinInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java

@@ -420,7 +420,8 @@ public class LensInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
   

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java

@@ -43,13 +43,17 @@ import javax.net.ssl.SSLContext;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.security.KeyStore;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.ConcurrentHashMap;
 
+
+
 /**
  * Base class for livy interpreters.
  */
@@ -68,9 +72,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   protected LivyVersion livyVersion;
   private RestTemplate restTemplate;
 
-  // keep tracking the mapping between paragraphId and statementId, so that we can cancel the
-  // statement after we execute it.
-  private ConcurrentHashMap<String, Integer> paragraphId2StmtIdMap = new ConcurrentHashMap<>();
+  Set<Object> paragraphsToCancel = Collections.newSetFromMap(
+      new ConcurrentHashMap<Object, Boolean>());
   private ConcurrentHashMap<String, Integer> paragraphId2StmtProgressMap =
       new ConcurrentHashMap<>();
 
@@ -163,21 +166,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
   @Override
   public void cancel(InterpreterContext context) {
-    if (livyVersion.isCancelSupported()) {
-      String paraId = context.getParagraphId();
-      Integer stmtId = paragraphId2StmtIdMap.get(paraId);
-      try {
-        if (stmtId != null) {
-          cancelStatement(stmtId);
-        }
-      } catch (LivyException e) {
-        LOGGER.error("Fail to cancel statement " + stmtId + " for paragraph " + paraId, e);
-      } finally {
-        paragraphId2StmtIdMap.remove(paraId);
-      }
-    } else {
-      LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
-    }
+    paragraphsToCancel.add(context.getParagraphId());
+    LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation.");
   }
 
   @Override
@@ -261,11 +251,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         }
         stmtInfo = executeStatement(new ExecuteRequest(code));
       }
-      if (paragraphId != null) {
-        paragraphId2StmtIdMap.put(paragraphId, stmtInfo.id);
-      }
       // pull the statement status
       while (!stmtInfo.isAvailable()) {
+        if (paragraphId != null && paragraphsToCancel.contains(paragraphId)) {
+          cancel(stmtInfo.id, paragraphId);
+          return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled");
+        }
         try {
           Thread.sleep(pullStatusInterval);
         } catch (InterruptedException e) {
@@ -285,12 +276,29 @@ public abstract class BaseLivyInterprereter extends Interpreter {
       }
     } finally {
       if (paragraphId != null) {
-        paragraphId2StmtIdMap.remove(paragraphId);
         paragraphId2StmtProgressMap.remove(paragraphId);
+        paragraphsToCancel.remove(paragraphId);
       }
     }
   }
 
+  private void cancel(int id, String paragraphId) {
+    if (livyVersion.isCancelSupported()) {
+      try {
+        LOGGER.info("Cancelling statement " + id);
+        cancelStatement(id);
+      } catch (LivyException e) {
+        LOGGER.error("Fail to cancel statement " + id + " for paragraph " + paragraphId, e);
+      }
+      finally {
+        paragraphsToCancel.remove(paragraphId);
+      }
+    } else {
+      LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
+      paragraphsToCancel.clear();
+    }
+  }
+
   protected LivyVersion getLivyVersion() throws LivyException {
     return new LivyVersion((LivyVersionResponse.fromJson(callRestAPI("/version", "GET")).version));
   }
@@ -371,7 +379,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
       if (displayAppInfo) {
         InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
-        interpreterResult.add(InterpreterResult.Type.TEXT, result);
+        interpreterResult.add(result);
         String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
             + "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
             + sessionInfo.webUIAddress + "</a>";

livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java

@@ -229,6 +229,11 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
     }
   }
 
+  @Override
+  public void cancel(InterpreterContext context) {
+    sparkInterpreter.cancel(context);
+  }
+
   @Override
   public void close() {
     this.sparkInterpreter.close();

livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java

@@ -145,6 +145,13 @@ public class LivyInterpreterIT {
         assertTrue(result.message().get(0).getData().contains("defined object Person"));
       }
 
+      // html output
+      String htmlCode = "println(\"%html <h1> hello </h1>\")";
+      result = sparkInterpreter.interpret(htmlCode, context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(1, result.message().size());
+      assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
+
       // error
       result = sparkInterpreter.interpret("println(a)", context);
       assertEquals(InterpreterResult.Code.ERROR, result.code());
@@ -162,9 +169,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }
@@ -306,6 +313,88 @@ public class LivyInterpreterIT {
     }
   }
 
+
+  @Test
+  public void testSparkSQLCancellation() {
+    if (!checkPreCondition()) {
+      return;
+    }
+    InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+    interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+    LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
+    sparkInterpreter.setInterpreterGroup(interpreterGroup);
+    interpreterGroup.get("session_1").add(sparkInterpreter);
+    AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+    MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+    InterpreterOutput output = new InterpreterOutput(outputListener);
+    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+        "title", "text", authInfo, null, null, null, null, null, output);
+    sparkInterpreter.open();
+
+    final LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
+    interpreterGroup.get("session_1").add(sqlInterpreter);
+    sqlInterpreter.setInterpreterGroup(interpreterGroup);
+    sqlInterpreter.open();
+
+    try {
+      // detect spark version
+      InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(1, result.message().size());
+
+      boolean isSpark2 = isSpark2(sparkInterpreter, context);
+
+      // test DataFrame api
+      if (!isSpark2) {
+        result = sparkInterpreter.interpret(
+            "val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
+        assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+        assertEquals(1, result.message().size());
+        assertTrue(result.message().get(0).getData()
+            .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
+      } else {
+        result = sparkInterpreter.interpret(
+            "val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
+        assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+        assertEquals(1, result.message().size());
+        assertTrue(result.message().get(0).getData()
+            .contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
+      }
+      sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
+
+      // cancel
+      if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) {
+        Thread cancelThread = new Thread() {
+          @Override
+          public void run() {
+            sqlInterpreter.cancel(context);
+          }
+        };
+        cancelThread.start();
+        //sleep so that cancelThread performs a cancel.
+        try {
+          Thread.sleep(1);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        result = sqlInterpreter
+            .interpret("select count(1) from df", context);
+        if (result.code().equals(InterpreterResult.Code.ERROR)) {
+          String message = result.message().get(0).getData();
+          // 2 possibilities, sometimes livy doesn't return the real cancel exception
+          assertTrue(message.contains("cancelled part of cancelled job group") ||
+              message.contains("Job is cancelled"));
+        }
+      }
+    } catch (LivyException e) {
+    } finally {
+      sparkInterpreter.close();
+      sqlInterpreter.close();
+    }
+  }
+
   @Test
   public void testStringWithTruncation() {
     if (!checkPreCondition()) {
@@ -495,9 +584,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }
@@ -544,8 +633,15 @@ public class LivyInterpreterIT {
       InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(2, result.message().size());
-
       assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
+
+      // html output
+      String htmlCode = "println(\"%html <h1> hello </h1>\")";
+      result = sparkInterpreter.interpret(htmlCode, context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertEquals(2, result.message().size());
+      assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
+
     } finally {
       sparkInterpreter.close();
     }
@@ -586,9 +682,9 @@ public class LivyInterpreterIT {
         Thread cancelThread = new Thread() {
           @Override
           public void run() {
-            // invoke cancel after 3 seconds to wait job starting
+            // invoke cancel after 1 millisecond to wait job starting
             try {
-              Thread.sleep(3000);
+              Thread.sleep(1);
             } catch (InterruptedException e) {
               e.printStackTrace();
             }

markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java

@@ -124,7 +124,8 @@ public class Markdown extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

pig/pom.xml

@@ -47,6 +47,12 @@
             <artifactId>zeppelin-interpreter</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

pom.xml

@@ -93,10 +93,12 @@
     <log4j.version>1.2.17</log4j.version>
     <libthrift.version>0.9.2</libthrift.version>
     <gson.version>2.2</gson.version>
+    <gson-extras.version>0.2.1</gson-extras.version>
     <guava.version>15.0</guava.version>
     <jetty.version>9.2.15.v20160210</jetty.version>
     <httpcomponents.core.version>4.3.3</httpcomponents.core.version>
     <httpcomponents.client.version>4.3.6</httpcomponents.client.version>
+    <httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
     <commons.lang.version>2.5</commons.lang.version>
     <commons.configuration.version>1.9</commons.configuration.version>
     <commons.codec.version>1.5</commons.codec.version>
@@ -173,6 +175,12 @@
         <version>${httpcomponents.client.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpasyncclient</artifactId>
+        <version>${httpcomponents.asyncclient.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>commons-lang</groupId>
         <artifactId>commons-lang</artifactId>
@@ -185,6 +193,12 @@
         <version>${gson.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.danilopianini</groupId>
+        <artifactId>gson-extras</artifactId>
+        <version>${gson-extras.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>commons-configuration</groupId>
         <artifactId>commons-configuration</artifactId>
@@ -925,39 +939,39 @@
               <exclude>**/interpreter-setting.json</exclude>
               <exclude>**/constants.json</exclude>
               <exclude>scripts/**</exclude>
-              <exclude>**/**/*.log</exclude> 
-              <exclude>**/**/logs/**</exclude> 
+              <exclude>**/**/*.log</exclude>
+              <exclude>**/**/logs/**</exclude>
 
-              <!-- bundled from zeppelin-web --> 
-              <exclude>**/test/karma.conf.js</exclude> 
-              <exclude>**/test/spec/**</exclude>      
-              <exclude>**/.babelrc</exclude> 
-              <exclude>**/.bowerrc</exclude> 
-              <exclude>.editorconfig</exclude> 
-              <exclude>.eslintrc</exclude> 
-              <exclude>**/.tmp/**</exclude> 
-              <exclude>**/target/**</exclude> 
-              <exclude>**/node/**</exclude> 
-              <exclude>**/node_modules/**</exclude> 
-              <exclude>**/bower_components/**</exclude> 
-              <exclude>**/dist/**</exclude> 
-              <exclude>**/.buildignore</exclude> 
-              <exclude>**/.npmignore</exclude> 
-              <exclude>**/.jshintrc</exclude> 
-              <exclude>**/yarn.lock</exclude> 
-              <exclude>**/bower.json</exclude> 
-              <exclude>**/src/fonts/Patua-One*</exclude> 
-              <exclude>**/src/fonts/patua-one*</exclude> 
-              <exclude>**/src/fonts/Roboto*</exclude> 
-              <exclude>**/src/fonts/roboto*</exclude> 
-              <exclude>**/src/fonts/fontawesome*</exclude> 
-              <exclude>**/src/fonts/font-awesome*</exclude> 
-              <exclude>**/src/styles/font-awesome*</exclude> 
-              <exclude>**/src/fonts/Simple-Line*</exclude> 
-              <exclude>**/src/fonts/simple-line*</exclude> 
-              <exclude>**/src/fonts/Source-Code-Pro*</exclude> 
+              <!-- bundled from zeppelin-web -->
+              <exclude>**/test/karma.conf.js</exclude>
+              <exclude>**/test/spec/**</exclude>
+              <exclude>**/.babelrc</exclude>
+              <exclude>**/.bowerrc</exclude>
+              <exclude>.editorconfig</exclude>
+              <exclude>.eslintrc</exclude>
+              <exclude>**/.tmp/**</exclude>
+              <exclude>**/target/**</exclude>
+              <exclude>**/node/**</exclude>
+              <exclude>**/node_modules/**</exclude>
+              <exclude>**/bower_components/**</exclude>
+              <exclude>**/dist/**</exclude>
+              <exclude>**/.buildignore</exclude>
+              <exclude>**/.npmignore</exclude>
+              <exclude>**/.jshintrc</exclude>
+              <exclude>**/yarn.lock</exclude>
+              <exclude>**/bower.json</exclude>
+              <exclude>**/src/fonts/Patua-One*</exclude>
+              <exclude>**/src/fonts/patua-one*</exclude>
+              <exclude>**/src/fonts/Roboto*</exclude>
+              <exclude>**/src/fonts/roboto*</exclude>
+              <exclude>**/src/fonts/fontawesome*</exclude>
+              <exclude>**/src/fonts/font-awesome*</exclude>
+              <exclude>**/src/styles/font-awesome*</exclude>
+              <exclude>**/src/fonts/Simple-Line*</exclude>
+              <exclude>**/src/fonts/simple-line*</exclude>
+              <exclude>**/src/fonts/Source-Code-Pro*</exclude>
               <exclude>**/src/fonts/source-code-pro*</exclude>
-              <exclude>**/src/**/**.test.js</exclude>               
+              <exclude>**/src/**/**.test.js</exclude>
 
               <!-- from SQLLine 1.0.2, see ZEPPELIN-2135 -->
               <exclude>**/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java</exclude>
@@ -1007,24 +1021,24 @@
               <exclude>**/package.json</exclude>
 
               <!-- compiled R packages (binaries) -->
-              <exclude>**/R/lib/**</exclude> 
-              <exclude>**/r/lib/**</exclude> 
- 
-               <!--R-related files with alternative licenses--> 
+              <exclude>**/R/lib/**</exclude>
+              <exclude>**/r/lib/**</exclude>
 
-              <exclude>**/R/rzeppelin/R/globals.R</exclude> 
-              <exclude>**/R/rzeppelin/R/common.R</exclude> 
-              <exclude>**/R/rzeppelin/R/protocol.R</exclude> 
-              <exclude>**/R/rzeppelin/R/rServer.R</exclude> 
-              <exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude> 
-              <exclude>**/R/rzeppelin/R/zzz.R</exclude> 
-              <exclude>**/src/main/scala/scala/Console.scala</exclude> 
-              <exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude> 
-              <exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude> 
- 
-               <!--The following files are mechanical--> 
-              <exclude>**/R/rzeppelin/DESCRIPTION</exclude> 
-              <exclude>**/R/rzeppelin/NAMESPACE</exclude> 
+               <!--R-related files with alternative licenses-->
+
+              <exclude>**/R/rzeppelin/R/globals.R</exclude>
+              <exclude>**/R/rzeppelin/R/common.R</exclude>
+              <exclude>**/R/rzeppelin/R/protocol.R</exclude>
+              <exclude>**/R/rzeppelin/R/rServer.R</exclude>
+              <exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude>
+              <exclude>**/R/rzeppelin/R/zzz.R</exclude>
+              <exclude>**/src/main/scala/scala/Console.scala</exclude>
+              <exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
+              <exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
+
+               <!--The following files are mechanical-->
+              <exclude>**/R/rzeppelin/DESCRIPTION</exclude>
+              <exclude>**/R/rzeppelin/NAMESPACE</exclude>
             </excludes>
           </configuration>
 

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -222,7 +222,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
-      registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
+      registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
     }
     // Add matplotlib display hook
     try {
@@ -435,7 +435,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java

@@ -87,7 +87,8 @@ public class PythonInterpreterPandasSql extends Interpreter {
     LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
     Interpreter python = getPythonInterpreter();
 
-    return python.interpret("z.show(pysqldf('" + st + "'))\nz._displayhook()", context);
+    return python.interpret(
+            "__zeppelin__.show(pysqldf('" + st + "'))\n__zeppelin__._displayhook()", context);
   }
 
   @Override

python/src/main/resources/python/zeppelin_python.py

@@ -53,7 +53,7 @@ class PyZeppelinContext(object):
 
   def __init__(self, z):
     self.z = z
-    self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
+    self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
     self.javaList = gateway.jvm.java.util.ArrayList
     self.max_result = 1000
     self._displayhook = lambda *args: None
@@ -195,6 +195,7 @@ host = "127.0.0.1"
 if len(sys.argv) >= 3:
   host = sys.argv[2]
 
+_zcUserQueryNameSpace = {}
 client = GatewayClient(address=host, port=int(sys.argv[1]))
 
 #gateway = JavaGateway(client, auto_convert = True)
@@ -204,8 +205,11 @@ intp = gateway.entry_point
 intp.onPythonScriptInitialized(os.getpid())
 
 java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
-z = PyZeppelinContext(intp)
-z._setup_matplotlib()
+z = __zeppelin__ = PyZeppelinContext(intp)
+__zeppelin__._setup_matplotlib()
+
+_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
+_zcUserQueryNameSpace["z"] = z
 
 output = Logger()
 sys.stdout = output
@@ -227,7 +231,7 @@ while True :
       global_hook = None
 
     try:
-      user_hook = z.getHook('post_exec')
+      user_hook = __zeppelin__.getHook('post_exec')
     except:
       user_hook = None
       
@@ -263,17 +267,17 @@ while True :
         for node in to_run_exec:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_single:
           mod = ast.Interactive([node])
           code = compile(mod, '<stdin>', 'single')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_hooks:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
       except:
         raise Exception(traceback.format_exc())
 

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java

@@ -106,6 +106,19 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
     assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
  }
 
+  @Test
+  public void testRedefinitionZeppelinContext() {
+    String pyRedefinitionCode = "z = 1\n";
+    String pyRestoreCode = "z = __zeppelin__\n";
+    String pyValidCode = "z.input(\"test\")\n";
+
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRedefinitionCode, context).code());
+    assertEquals(InterpreterResult.Code.ERROR, pythonInterpreter.interpret(pyValidCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRestoreCode, context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
+  }
+
   @Override
   public void onUpdateAll(InterpreterOutput out) {
 

r/src/main/java/org/apache/zeppelin/rinterpreter/KnitR.java

@@ -77,8 +77,9 @@ public class KnitR extends Interpreter implements WrappedInterpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
-    List completion = intp.completion(s, i);
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
+    List completion = intp.completion(s, i, interpreterContext);
     return completion;
   }
 

r/src/main/java/org/apache/zeppelin/rinterpreter/RRepl.java

@@ -77,8 +77,9 @@ public class RRepl extends Interpreter implements WrappedInterpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String s, int i) {
-    List completion = intp.completion(s, i);
+  public List<InterpreterCompletion> completion(String s, int i,
+      InterpreterContext interpreterContext) {
+    List completion = intp.completion(s, i, interpreterContext);
     return completion;
   }
 

scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java

@@ -270,7 +270,8 @@ public class ScaldingInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return NO_COMPLETION;
   }
 

shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java

@@ -138,7 +138,8 @@ public class ShellInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

spark-dependencies/pom.xml

@@ -344,6 +344,19 @@
       </exclusions>
     </dependency>
 
+    <!-- yarn (not supported for Spark v1.5.0 or higher) -->
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+      <version>${spark.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+      <version>${yarn.version}</version>
+    </dependency>
+
   </dependencies>
 
   <profiles>
@@ -810,23 +823,6 @@
       </repositories>
     </profile>
 
-    <profile>
-      <id>yarn</id>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-          <version>${spark.version}</version>
-        </dependency>
-
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-api</artifactId>
-          <version>${yarn.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-
   </profiles>
 
   <build>

spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java

@@ -33,6 +33,8 @@ import java.util.Properties;
 
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.spark.repl.SparkILoop;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -284,7 +286,8 @@ public class DepInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (Utils.isScala2_10()) {
       ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer");
       Candidates ret = c.complete(buf, cursor);
@@ -293,7 +296,7 @@ public class DepInterpreter extends Interpreter {
       List<InterpreterCompletion> completions = new LinkedList<>();
 
       for (String candidate : candidates) {
-        completions.add(new InterpreterCompletion(candidate, candidate));
+        completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
       }
 
       return completions;

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -42,6 +42,7 @@ import org.apache.commons.exec.ExecuteResultHandler;
 import org.apache.commons.exec.ExecuteWatchdog;
 import org.apache.commons.exec.PumpStreamHandler;
 import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SQLContext;
@@ -113,7 +114,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
-      registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
+      registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
     }
     DepInterpreter depInterpreter = getDepInterpreter();
 
@@ -390,9 +391,9 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
       return new InterpreterResult(Code.ERROR, errorMessage);
     }
     String jobGroup = Utils.buildJobGroupId(context);
-    ZeppelinContext z = sparkInterpreter.getZeppelinContext();
-    z.setInterpreterContext(context);
-    z.setGui(context.getGui());
+    ZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
+    __zeppelin__.setInterpreterContext(context);
+    __zeppelin__.setGui(context.getGui());
     pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup);
     statementOutput = null;
 
@@ -457,7 +458,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (buf.length() < cursor) {
       cursor = buf.length();
     }
@@ -508,7 +510,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
     List<InterpreterCompletion> results = new LinkedList<>();
     for (String name: completionList) {
-      results.add(new InterpreterCompletion(name, name));
+      results.add(new InterpreterCompletion(name, name, StringUtils.EMPTY));
     }
     return results;
   }

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -246,7 +246,7 @@ public class SparkInterpreter extends Interpreter {
    */
   private boolean hiveClassesArePresent() {
     try {
-      this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState");
+      this.getClass().forName("org.apache.spark.sql.hive.execution.InsertIntoHiveTable");
       this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf");
       return true;
     } catch (ClassNotFoundException | NoClassDefFoundError e) {
@@ -1068,7 +1068,8 @@ public class SparkInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     if (completer == null) {
       logger.warn("Can't find completer");
       return new LinkedList<>();
@@ -1090,7 +1091,7 @@ public class SparkInterpreter extends Interpreter {
     List<InterpreterCompletion> completions = new LinkedList<>();
 
     for (String candidate : candidates) {
-      completions.add(new InterpreterCompletion(candidate, candidate));
+      completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
     }
 
     return completions;

spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java

@@ -212,7 +212,8 @@ public class SparkRInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return new ArrayList<>();
   }
 

spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java

@@ -177,7 +177,8 @@ public class SparkSqlInterpreter extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java

@@ -40,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.AngularObjectWatcher;
 import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.display.Input.ParamOption;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterException;
@@ -114,14 +114,33 @@ public class ZeppelinContext {
   public SQLContext sqlContext;
   private GUI gui;
 
+  /**
+   * @deprecated use z.textbox instead
+   *
+   */
+  @Deprecated
   @ZeppelinApi
   public Object input(String name) {
-    return input(name, "");
+    return textbox(name);
+  }
+
+  /**
+   * @deprecated use z.textbox instead
+   */
+  @Deprecated
+  @ZeppelinApi
+  public Object input(String name, Object defaultValue) {
+    return textbox(name, defaultValue.toString());
   }
 
   @ZeppelinApi
-  public Object input(String name, Object defaultValue) {
-    return gui.input(name, defaultValue);
+  public Object textbox(String name) {
+    return textbox(name, "");
+  }
+
+  @ZeppelinApi
+  public Object textbox(String name, String defaultValue) {
+    return gui.textbox(name, defaultValue);
   }
 
   @ZeppelinApi
@@ -136,7 +155,7 @@ public class ZeppelinContext {
   }
 
   @ZeppelinApi
-  public scala.collection.Iterable<Object> checkbox(String name,
+  public scala.collection.Seq<Object> checkbox(String name,
       scala.collection.Iterable<Tuple2<Object, String>> options) {
     List<Object> allChecked = new LinkedList<>();
     for (Tuple2<Object, String> option : asJavaIterable(options)) {
@@ -146,11 +165,12 @@ public class ZeppelinContext {
   }
 
   @ZeppelinApi
-  public scala.collection.Iterable<Object> checkbox(String name,
+  public scala.collection.Seq<Object> checkbox(String name,
       scala.collection.Iterable<Object> defaultChecked,
       scala.collection.Iterable<Tuple2<Object, String>> options) {
-    return collectionAsScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked),
-      tuplesToParamOptions(options)));
+    return scala.collection.JavaConversions.asScalaBuffer(
+        gui.checkbox(name, asJavaCollection(defaultChecked),
+            tuplesToParamOptions(options))).toSeq();
   }
 
   private ParamOption[] tuplesToParamOptions(
@@ -311,7 +331,7 @@ public class ZeppelinContext {
    */
   @ZeppelinApi
   public void run(String noteId, String paragraphId) {
-    run(noteId, paragraphId, interpreterContext);
+    run(noteId, paragraphId, interpreterContext, true);
   }
 
   /**
@@ -320,8 +340,27 @@ public class ZeppelinContext {
    */
   @ZeppelinApi
   public void run(String paragraphId) {
+    run(paragraphId, true);
+  }
+
+  /**
+   * Run paragraph by id
+   * @param paragraphId
+   * @param checkCurrentParagraph
+   */
+  @ZeppelinApi
+  public void run(String paragraphId, boolean checkCurrentParagraph) {
     String noteId = interpreterContext.getNoteId();
-    run(noteId, paragraphId, interpreterContext);
+    run(noteId, paragraphId, interpreterContext, checkCurrentParagraph);
+  }
+
+  /**
+   * Run paragraph by id
+   * @param noteId
+   */
+  @ZeppelinApi
+  public void run(String noteId, String paragraphId, InterpreterContext context) {
+    run(noteId, paragraphId, context, true);
   }
 
   /**
@@ -330,8 +369,9 @@ public class ZeppelinContext {
    * @param context
    */
   @ZeppelinApi
-  public void run(String noteId, String paragraphId, InterpreterContext context) {
-    if (paragraphId.equals(context.getParagraphId())) {
+  public void run(String noteId, String paragraphId, InterpreterContext context,
+                  boolean checkCurrentParagraph) {
+    if (paragraphId.equals(context.getParagraphId()) && checkCurrentParagraph) {
       throw new InterpreterException("Can not run current Paragraph");
     }
 
@@ -411,24 +451,50 @@ public class ZeppelinContext {
    */
   @ZeppelinApi
   public void run(int idx) {
+    run(idx, true);
+  }
+
+  /**
+   *
+   * @param idx  paragraph index
+   * @param checkCurrentParagraph  check whether you call this run method in the current paragraph.
+   * Set it to false only when you are sure you are not invoking this method to run current
+   * paragraph. Otherwise you would run current paragraph in infinite loop.
+   */
+  public void run(int idx, boolean checkCurrentParagraph) {
     String noteId = interpreterContext.getNoteId();
-    run(noteId, idx, interpreterContext);
+    run(noteId, idx, interpreterContext, checkCurrentParagraph);
   }
 
   /**
    * Run paragraph at index
+   * @param noteId
    * @param idx index starting from 0
    * @param context interpreter context
    */
   public void run(String noteId, int idx, InterpreterContext context) {
+    run(noteId, idx, context, true);
+  }
+
+  /**
+   *
+   * @param noteId
+   * @param idx  paragraph index
+   * @param context interpreter context
+   * @param checkCurrentParagraph check whether you call this run method in the current paragraph.
+   * Set it to false only when you are sure you are not invoking this method to run current
+   * paragraph. Otherwise you would run current paragraph in infinite loop.
+   */
+  public void run(String noteId, int idx, InterpreterContext context,
+                  boolean checkCurrentParagraph) {
     List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
     if (idx >= runners.size()) {
       throw new InterpreterException("Index out of bound");
     }
 
     InterpreterContextRunner runner = runners.get(idx);
-    if (runner.getParagraphId().equals(context.getParagraphId())) {
-      throw new InterpreterException("Can not run current Paragraph");
+    if (runner.getParagraphId().equals(context.getParagraphId()) && checkCurrentParagraph) {
+      throw new InterpreterException("Can not run current Paragraph: " + runner.getParagraphId());
     }
 
     runner.run();

spark/src/main/resources/python/zeppelin_pyspark.py

@@ -97,13 +97,15 @@ class PyZeppelinContext(dict):
 
   def checkbox(self, name, options, defaultChecked=None):
     if defaultChecked is None:
-      defaultChecked = list(map(lambda items: items[0], options))
+      defaultChecked = []
     optionTuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options))
     optionIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(optionTuples)
     defaultCheckedIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(defaultChecked)
-
-    checkedIterables = self.z.checkbox(name, defaultCheckedIterables, optionIterables)
-    return gateway.jvm.scala.collection.JavaConversions.asJavaCollection(checkedIterables)
+    checkedItems = gateway.jvm.scala.collection.JavaConversions.seqAsJavaList(self.z.checkbox(name, defaultCheckedIterables, optionIterables))
+    result = []
+    for checkedItem in checkedItems:
+      result.append(checkedItem)
+    return result;
 
   def registerHook(self, event, cmd, replName=None):
     if replName is None:
@@ -271,19 +273,37 @@ else:
 
 java_import(gateway.jvm, "scala.Tuple2")
 
+_zcUserQueryNameSpace = {}
+
 jconf = intp.getSparkConf()
 conf = SparkConf(_jvm = gateway.jvm, _jconf = jconf)
-sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
-if sparkVersion.isSpark2():
-  spark = SparkSession(sc, intp.getSparkSession())
-  sqlc = spark._wrapped
-else:
-  sqlc = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
-sqlContext = sqlc
+sc = _zsc_ = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
+_zcUserQueryNameSpace["_zsc_"] = _zsc_
+_zcUserQueryNameSpace["sc"] = sc
 
-completion = PySparkCompletion(intp)
-z = PyZeppelinContext(intp.getZeppelinContext())
-z._setup_matplotlib()
+if sparkVersion.isSpark2():
+  spark = __zSpark__ = SparkSession(sc, intp.getSparkSession())
+  sqlc = __zSqlc__ = __zSpark__._wrapped
+  _zcUserQueryNameSpace["sqlc"] = sqlc
+  _zcUserQueryNameSpace["__zSqlc__"] = __zSqlc__
+  _zcUserQueryNameSpace["spark"] = spark
+  _zcUserQueryNameSpace["__zSpark__"] = __zSpark__
+else:
+  sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
+  _zcUserQueryNameSpace["sqlc"] = sqlc
+  _zcUserQueryNameSpace["__zSqlc__"] = sqlc
+
+sqlContext = __zSqlc__
+_zcUserQueryNameSpace["sqlContext"] = sqlContext
+
+completion = __zeppelin_completion__ = PySparkCompletion(intp)
+_zcUserQueryNameSpace["completion"] = completion
+_zcUserQueryNameSpace["__zeppelin_completion__"] = __zeppelin_completion__
+
+z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
+__zeppelin__._setup_matplotlib()
+_zcUserQueryNameSpace["z"] = z
+_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
 
 while True :
   req = intp.getStatements()
@@ -299,7 +319,7 @@ while True :
       global_hook = None
       
     try:
-      user_hook = z.getHook('post_exec')
+      user_hook = __zeppelin__.getHook('post_exec')
     except:
       user_hook = None
       
@@ -334,17 +354,17 @@ while True :
         for node in to_run_exec:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
 
         for node in to_run_single:
           mod = ast.Interactive([node])
           code = compile(mod, '<stdin>', 'single')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
           
         for node in to_run_hooks:
           mod = ast.Module([node])
           code = compile(mod, '<stdin>', 'exec')
-          exec(code)
+          exec(code, _zcUserQueryNameSpace)
       except:
         raise Exception(traceback.format_exc())
 

spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java

@@ -118,11 +118,26 @@ public class PySparkInterpreterTest {
   @Test
   public void testCompletion() {
     if (getSparkVersionNumber() > 11) {
-      List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length());
+      List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length(), null);
       assertTrue(completions.size() > 0);
     }
   }
 
+  @Test
+  public void testRedefinitionZeppelinContext() {
+    if (getSparkVersionNumber() > 11) {
+      String redefinitionCode = "z = 1\n";
+      String restoreCode = "z = __zeppelin__\n";
+      String validCode = "z.input(\"test\")\n";
+
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(redefinitionCode, context).code());
+      assertEquals(InterpreterResult.Code.ERROR, pySparkInterpreter.interpret(validCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(restoreCode, context).code());
+      assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
+    }
+  }
+
   private class infinityPythonJob implements Runnable {
     @Override
     public void run() {

spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java

@@ -301,7 +301,7 @@ public class SparkInterpreterTest {
 
   @Test
   public void testCompletion() {
-    List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length());
+    List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length(), null);
     assertTrue(completions.size() > 0);
   }
 

zeppelin-distribution/src/bin_license/LICENSE

@@ -229,7 +229,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (The MIT License) Json3 v3.3.1 (http://bestiejs.github.io/json3) - https://github.com/bestiejs/json3/blob/v3.3.1/LICENSE
     (The MIT License) es5-shim v3.1.0 (https://github.com/es-shims/es5-shim) - https://github.com/es-shims/es5-shim/blob/v3.1.0/LICENSE
     (The MIT License) bootstrap v3.2.0 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.2.0/LICENSE
-    (The MIT License) UI Bootstrap v0.13.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/0.13.0/LICENSE
+    (The MIT License) UI Bootstrap v2.5.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/2.5.0/LICENSE
     (The MIT License) bootstrap3-dialog v1.34.7 (https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7) - https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7
     (The MIT License) Angular Websocket v1.0.13 (http://angularclass.github.io/angular-websocket/) - https://github.com/AngularClass/angular-websocket/blob/v1.0.13/LICENSE
     (The MIT License) UI.Ace v0.1.1 (http://angularclass.github.io/angular-websocket/) - https://github.com/angular-ui/ui-ace/blob/master/LICENSE

zeppelin-interpreter/pom.xml

@@ -43,6 +43,7 @@
     <aether.version>1.12</aether.version>
     <maven.aeither.provider.version>3.0.3</maven.aeither.provider.version>
     <wagon.version>1.0</wagon.version>
+    <jline.version>2.12.1</jline.version>
 
     <!--plugin versions-->
     <plugin.shade.version>2.3</plugin.shade.version>
@@ -60,6 +61,11 @@
       <artifactId>gson</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.danilopianini</groupId>
+      <artifactId>gson-extras</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-exec</artifactId>
@@ -202,6 +208,17 @@
       <version>${wagon.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>jline</groupId>
+      <artifactId>jline</artifactId>
+      <version>${jline.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CompletionType.java

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.zeppelin.completer;
+
+/**
+ * Types of completion
+ */
+public enum CompletionType {
+  schema,
+  table,
+  column,
+  setting,
+  command,
+  keyword,
+  path
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/StringsCompleter.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.zeppelin.completer;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import jline.console.completer.Completer;
+import jline.internal.Preconditions;
+
+/**
+ * Case-insensitive completer for a set of strings.
+ */
+public class StringsCompleter implements Completer {
+  private final SortedSet<String> strings = new TreeSet<String>(new Comparator<String>() {
+    @Override
+    public int compare(String o1, String o2) {
+      return o1.compareToIgnoreCase(o2);
+    }
+  });
+
+  public StringsCompleter() {
+  }
+
+  public StringsCompleter(final Collection<String> strings) {
+    Preconditions.checkNotNull(strings);
+    getStrings().addAll(strings);
+  }
+
+  public Collection<String> getStrings() {
+    return strings;
+  }
+
+  public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
+    return completeCollection(buffer, cursor, candidates);
+  }
+
+  public int complete(final String buffer, final int cursor, final Set<CharSequence> candidates) {
+    return completeCollection(buffer, cursor, candidates);
+  }
+
+  private int completeCollection(final String buffer, final int cursor,
+      final Collection<CharSequence> candidates) {
+    Preconditions.checkNotNull(candidates);
+    if (buffer == null) {
+      candidates.addAll(strings);
+    } else {
+      String bufferTmp = buffer.toUpperCase();
+      for (String match : strings.tailSet(buffer)) {
+        String matchTmp = match.toUpperCase();
+        if (!matchTmp.startsWith(bufferTmp)) {
+          break;
+        }
+
+        candidates.add(match);
+      }
+    }
+
+    return candidates.isEmpty() ? -1 : 0;
+  }
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java

@@ -34,6 +34,8 @@ import org.slf4j.LoggerFactory;
  * @param <T>
  */
 public class AngularObject<T> {
+  private static final Logger LOGGER = LoggerFactory.getLogger(AngularObject.class);
+
   private String name;
   private T object;
   
@@ -172,7 +174,7 @@ public class AngularObject<T> {
     if (emit) {
       emit();
     }
-
+    LOGGER.debug("Update angular object: " + name + " with value: " + o);
     final Logger logger = LoggerFactory.getLogger(AngularObject.class);
     List<AngularObjectWatcher> ws = new LinkedList<>();
     synchronized (watchers) {

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/GUI.java

@@ -17,17 +17,27 @@
 
 package org.apache.zeppelin.display;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.apache.zeppelin.display.ui.CheckBox;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
+import org.apache.zeppelin.display.ui.Select;
+import org.apache.zeppelin.display.ui.TextBox;
+
 import java.io.Serializable;
 
 import java.util.*;
 
-import org.apache.zeppelin.display.Input.ParamOption;
 
 /**
  * Settings of a form.
  */
 public class GUI implements Serializable {
 
+  private static Gson gson = new GsonBuilder()
+      .registerTypeAdapterFactory(Input.TypeAdapterFactory)
+      .create();
+
   Map<String, Object> params = new HashMap<>(); // form parameters from client
   LinkedHashMap<String, Input> forms = new LinkedHashMap<>(); // form configuration
 
@@ -51,19 +61,29 @@ public class GUI implements Serializable {
     this.forms = forms;
   }
 
+  @Deprecated
+  public Object input(String id) {
+    return textbox(id, "");
+  }
+
+  @Deprecated
   public Object input(String id, Object defaultValue) {
+    return textbox(id, defaultValue.toString());
+  }
+
+  public Object textbox(String id, String defaultValue) {
     // first find values from client and then use default
     Object value = params.get(id);
     if (value == null) {
       value = defaultValue;
     }
 
-    forms.put(id, new Input(id, defaultValue, "input"));
+    forms.put(id, new TextBox(id, defaultValue));
     return value;
   }
 
-  public Object input(String id) {
-    return input(id, "");
+  public Object textbox(String id) {
+    return textbox(id, "");
   }
 
   public Object select(String id, Object defaultValue, ParamOption[] options) {
@@ -71,18 +91,18 @@ public class GUI implements Serializable {
     if (value == null) {
       value = defaultValue;
     }
-    forms.put(id, new Input(id, defaultValue, "select", options));
+    forms.put(id, new Select(id, defaultValue, options));
     return value;
   }
 
-  public Collection<Object> checkbox(String id, Collection<Object> defaultChecked,
+  public List<Object> checkbox(String id, Collection<Object> defaultChecked,
                                      ParamOption[] options) {
     Collection<Object> checked = (Collection<Object>) params.get(id);
     if (checked == null) {
       checked = defaultChecked;
     }
-    forms.put(id, new Input(id, defaultChecked, "checkbox", options));
-    Collection<Object> filtered = new LinkedList<>();
+    forms.put(id, new CheckBox(id, defaultChecked, options));
+    List<Object> filtered = new LinkedList<>();
     for (Object o : checked) {
       if (isValidOption(o, options)) {
         filtered.add(o);
@@ -103,4 +123,41 @@ public class GUI implements Serializable {
   public void clear() {
     this.forms = new LinkedHashMap<>();
   }
+
+  public String toJson() {
+    return gson.toJson(this);
+  }
+
+  public void convertOldInput() {
+    for (Map.Entry<String, Input> entry : forms.entrySet()) {
+      if (entry.getValue() instanceof OldInput) {
+        Input convertedInput = convertFromOldInput((OldInput) entry.getValue());
+        forms.put(entry.getKey(), convertedInput);
+      }
+    }
+  }
+
+  public static GUI fromJson(String json) {
+    GUI gui = gson.fromJson(json, GUI.class);
+    gui.convertOldInput();
+    return gui;
+  }
+
+  private Input convertFromOldInput(OldInput oldInput) {
+    Input convertedInput = null;
+
+    if (oldInput.options == null || oldInput instanceof OldInput.OldTextBox) {
+      convertedInput = new TextBox(oldInput.name, oldInput.defaultValue.toString());
+    } else if (oldInput instanceof OldInput.OldCheckBox) {
+      convertedInput = new CheckBox(oldInput.name, (List) oldInput.defaultValue, oldInput.options);
+    } else if (oldInput instanceof OldInput && oldInput.options != null) {
+      convertedInput = new Select(oldInput.name, oldInput.defaultValue, oldInput.options);
+    } else {
+      throw new RuntimeException("Can not convert this OldInput.");
+    }
+    convertedInput.setDisplayName(oldInput.getDisplayName());
+    convertedInput.setHidden(oldInput.isHidden());
+    convertedInput.setArgument(oldInput.getArgument());
+    return convertedInput;
+  }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java

@@ -18,6 +18,8 @@
 package org.apache.zeppelin.display;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.display.ui.*;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
 
 import java.io.Serializable;
 import java.util.*;
@@ -25,105 +27,43 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
- * Input type.
+ * Base class for dynamic forms. Also used as factory class of dynamic forms.
+ *
+ * @param <T>
  */
-public class Input implements Serializable {
-  /**
-   * Parameters option.
-   */
-  public static class ParamOption {
-    Object value;
-    String displayName;
+public class Input<T> implements Serializable {
 
-    public ParamOption(Object value, String displayName) {
-      super();
-      this.value = value;
-      this.displayName = displayName;
-    }
+  // @TODO(zjffdu). Use gson's RuntimeTypeAdapterFactory and remove the old input form support
+  // in future.
+  public static final RuntimeTypeAdapterFactory TypeAdapterFactory =
+      RuntimeTypeAdapterFactory.of(Input.class, "type")
+        .registerSubtype(TextBox.class, "TextBox")
+        .registerSubtype(Select.class, "Select")
+        .registerSubtype(CheckBox.class, "CheckBox")
+        .registerSubtype(OldInput.OldTextBox.class, "input")
+        .registerSubtype(OldInput.OldSelect.class, "select")
+        .registerSubtype(OldInput.OldCheckBox.class, "checkbox")
+        .registerSubtype(OldInput.class, null);
 
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) return true;
-      if (o == null || getClass() != o.getClass()) return false;
-
-      ParamOption that = (ParamOption) o;
-
-      if (value != null ? !value.equals(that.value) : that.value != null) return false;
-      return displayName != null ? displayName.equals(that.displayName) : that.displayName == null;
-
-    }
-
-    @Override
-    public int hashCode() {
-      int result = value != null ? value.hashCode() : 0;
-      result = 31 * result + (displayName != null ? displayName.hashCode() : 0);
-      return result;
-    }
-
-    public Object getValue() {
-      return value;
-    }
-
-    public void setValue(Object value) {
-      this.value = value;
-    }
-
-    public String getDisplayName() {
-      return displayName;
-    }
-
-    public void setDisplayName(String displayName) {
-      this.displayName = displayName;
-    }
+  protected String name;
+  protected String displayName;
+  protected T defaultValue;
+  protected boolean hidden;
+  protected String argument;
 
+  public Input() {
   }
 
-  String name;
-  String displayName;
-  String type;
-  String argument;
-  Object defaultValue;
-  ParamOption[] options;
-  boolean hidden;
-
-  public Input(String name, Object defaultValue, String type) {
-    this.name = name;
-    this.displayName = name;
-    this.defaultValue = defaultValue;
-    this.type = type;
-  }
-
-  public Input(String name, Object defaultValue, String type, ParamOption[] options) {
-    this.name = name;
-    this.displayName = name;
-    this.defaultValue = defaultValue;
-    this.type = type;
-    this.options = options;
-  }
-
-  public Input(String name, String displayName, String type, String argument, Object defaultValue,
-      ParamOption[] options, boolean hidden) {
-    super();
-    this.name = name;
-    this.displayName = displayName;
-    this.argument = argument;
-    this.type = type;
-    this.defaultValue = defaultValue;
-    this.options = options;
-    this.hidden = hidden;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    return name.equals(((Input) o).getName());
+  public boolean isHidden() {
+    return hidden;
   }
 
   public String getName() {
-    return name;
+    return this.name;
   }
 
-  public void setName(String name) {
-    this.name = name;
+  public T getDefaultValue() {
+    return defaultValue;
   }
 
   public String getDisplayName() {
@@ -134,41 +74,37 @@ public class Input implements Serializable {
     this.displayName = displayName;
   }
 
-  public String getType() {
-    return type;
+  public void setArgument(String argument) {
+    this.argument = argument;
   }
 
-  public void setType(String type) {
-    this.type = type;
+  public void setHidden(boolean hidden) {
+    this.hidden = hidden;
   }
 
-  public Object getDefaultValue() {
-    return defaultValue;
+  public String getArgument() {
+    return argument;
   }
 
-  public void setDefaultValue(Object defaultValue) {
-    this.defaultValue = defaultValue;
+  public static TextBox textbox(String name, String defaultValue) {
+    return new TextBox(name, defaultValue);
   }
 
-  public ParamOption[] getOptions() {
-    return options;
+  public static Select select(String name, Object defaultValue, ParamOption[] options) {
+    return new Select(name, defaultValue, options);
   }
 
-  public void setOptions(ParamOption[] options) {
-    this.options = options;
-  }
-
-  public boolean isHidden() {
-    return hidden;
+  public static CheckBox checkbox(String name, Object[] defaultChecked, ParamOption[] options) {
+    return new CheckBox(name, defaultChecked, options);
   }
 
   // Syntax of variables: ${TYPE:NAME=DEFAULT_VALUE1|DEFAULT_VALUE2|...,VALUE1|VALUE2|...}
   // Type is optional. Type may contain an optional argument with syntax: TYPE(ARG)
   // NAME and VALUEs may contain an optional display name with syntax: NAME(DISPLAY_NAME)
   // DEFAULT_VALUEs may not contain display name
-  // Examples:  ${age}                              input form without default value
-  //            ${age=3}                            input form with default value
-  //            ${age(Age)=3}                       input form with display name and default value
+  // Examples:  ${age}                              textbox form without default value
+  //            ${age=3}                            textbox form with default value
+  //            ${age(Age)=3}                       textbox form with display name and default value
   //            ${country=US(United States)|UK|JP}  select form with
   //            ${checkbox( or ):country(Country)=US|JP,US(United States)|UK|JP}
   //                                                checkbox form with " or " as delimiter: will be
@@ -282,7 +218,22 @@ public class Input implements Serializable {
 
     }
 
-    return new Input(varName, displayName, type, arg, defaultValue, paramOptions, hidden);
+    Input input = null;
+    if (type == null) {
+      if (paramOptions == null) {
+        input = new TextBox(varName, (String) defaultValue);
+      } else {
+        input = new Select(varName, defaultValue, paramOptions);
+      }
+    } else if (type.equals("checkbox")) {
+      input = new CheckBox(varName, (Object[]) defaultValue, paramOptions);
+    } else {
+      throw new RuntimeException("Could not recognize dynamic form with type: " + type);
+    }
+    input.setArgument(arg);
+    input.setDisplayName(displayName);
+    input.setHidden(hidden);
+    return input;
   }
 
   public static LinkedHashMap<String, Input> extractSimpleQueryForm(String script) {
@@ -314,11 +265,12 @@ public class Input implements Serializable {
       if (params.containsKey(input.name)) {
         value = params.get(input.name);
       } else {
-        value = input.defaultValue;
+        value = input.getDefaultValue();
       }
 
       String expanded;
       if (value instanceof Object[] || value instanceof Collection) {  // multi-selection
+        OptionInput optionInput = (OptionInput) input;
         String delimiter = input.argument;
         if (delimiter == null) {
           delimiter = DEFAULT_DELIMITER;
@@ -327,7 +279,7 @@ public class Input implements Serializable {
                 : Arrays.asList((Object[]) value);
         List<Object> validChecked = new LinkedList<>();
         for (Object o : checked) {  // filter out obsolete checked values
-          for (ParamOption option : input.getOptions()) {
+          for (ParamOption option : optionInput.getOptions()) {
             if (option.getValue().equals(o)) {
               validChecked.add(o);
               break;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/OldInput.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.display;
+
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
+
+/**
+ * Old Input type.
+ * The reason I still keep Old Input is for compatibility. There's one bug in the old input forms.
+ * There's 2 ways to create input forms: frontend & backend.
+ * The bug is in frontend. The type would not be set correctly when input form
+ * is created in frontend (Input.getInputForm).
+ */
+public class OldInput extends Input<Object> {
+
+  ParamOption[] options;
+
+  public OldInput() {}
+
+  public OldInput(String name, Object defaultValue) {
+    this.name = name;
+    this.displayName = name;
+    this.defaultValue = defaultValue;
+  }
+
+  public OldInput(String name, Object defaultValue, ParamOption[] options) {
+    this.name = name;
+    this.displayName = name;
+    this.defaultValue = defaultValue;
+    this.options = options;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    return name.equals(((OldInput) o).getName());
+  }
+
+  public ParamOption[] getOptions() {
+    return options;
+  }
+
+  public void setOptions(ParamOption[] options) {
+    this.options = options;
+  }
+
+  /**
+   *
+   */
+  public static class OldTextBox extends OldInput {
+    public OldTextBox(String name, Object defaultValue) {
+      super(name, defaultValue);
+    }
+  }
+
+  /**
+   *
+   */
+  public static class OldSelect extends OldInput {
+    public OldSelect(String name, Object defaultValue, ParamOption[] options) {
+      super(name, defaultValue, options);
+    }
+  }
+
+  /**
+   *
+   */
+  public static class OldCheckBox extends OldInput {
+    public OldCheckBox(String name, Object defaultValue, ParamOption[] options) {
+      super(name, defaultValue, options);
+    }
+  }
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/RuntimeTypeAdapterFactory.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.display;
+
+import com.google.gson.*;
+import com.google.gson.internal.Streams;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Copied from gson with minor changes to support old input forms
+ *
+ * @param <T>
+ */
+public class RuntimeTypeAdapterFactory<T> implements TypeAdapterFactory {
+  private final Class<?> baseType;
+  private final String typeFieldName;
+  private final Map<String, Class<?>> labelToSubtype = new LinkedHashMap<String, Class<?>>();
+  private final Map<Class<?>, String> subtypeToLabel = new LinkedHashMap<Class<?>, String>();
+
+  private RuntimeTypeAdapterFactory(Class<?> baseType, String typeFieldName) {
+    if (typeFieldName == null || baseType == null) {
+      throw new NullPointerException();
+    }
+    this.baseType = baseType;
+    this.typeFieldName = typeFieldName;
+  }
+
+  /**
+   * Creates a new runtime type adapter using for {@code baseType} using {@code
+   * typeFieldName} as the type field name. Type field names are case sensitive.
+   */
+  public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType, String typeFieldName) {
+    return new RuntimeTypeAdapterFactory<T>(baseType, typeFieldName);
+  }
+
+  /**
+   * Creates a new runtime type adapter for {@code baseType} using {@code "type"} as
+   * the type field name.
+   */
+  public static <T> RuntimeTypeAdapterFactory<T> of(Class<T> baseType) {
+    return new RuntimeTypeAdapterFactory<T>(baseType, "type");
+  }
+
+  /**
+   * Registers {@code type} identified by {@code label}. Labels are case
+   * sensitive.
+   *
+   * @throws IllegalArgumentException if either {@code type} or {@code label}
+   *     have already been registered on this type adapter.
+   */
+  public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type, String label) {
+    if (type == null) {
+      throw new NullPointerException();
+    }
+    if (subtypeToLabel.containsKey(type) || labelToSubtype.containsKey(label)) {
+      throw new IllegalArgumentException("types and labels must be unique");
+    }
+    labelToSubtype.put(label, type);
+    subtypeToLabel.put(type, label);
+    return this;
+  }
+
+  /**
+   * Registers {@code type} identified by its {@link Class#getSimpleName simple
+   * name}. Labels are case sensitive.
+   *
+   * @throws IllegalArgumentException if either {@code type} or its simple name
+   *     have already been registered on this type adapter.
+   */
+  public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type) {
+    return registerSubtype(type, type.getSimpleName());
+  }
+
+  public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
+    if (type.getRawType() != baseType) {
+      return null;
+    }
+
+    final Map<String, TypeAdapter<?>> labelToDelegate = new LinkedHashMap<String, TypeAdapter<?>>();
+    final Map<Class<?>, TypeAdapter<?>> subtypeToDelegate =
+        new LinkedHashMap<Class<?>, TypeAdapter<?>>();
+    for (Map.Entry<String, Class<?>> entry : labelToSubtype.entrySet()) {
+      TypeAdapter<?> delegate = gson.getDelegateAdapter(this, TypeToken.get(entry.getValue()));
+      labelToDelegate.put(entry.getKey(), delegate);
+      subtypeToDelegate.put(entry.getValue(), delegate);
+    }
+
+    return new TypeAdapter<R>() {
+      @Override public R read(JsonReader in) throws IOException {
+        JsonElement jsonElement = Streams.parse(in);
+        JsonElement labelJsonElement = jsonElement.getAsJsonObject().remove(typeFieldName);
+        String label = (labelJsonElement == null ? null : labelJsonElement.getAsString());
+        @SuppressWarnings("unchecked") // registration requires that subtype extends T
+            TypeAdapter<R> delegate = (TypeAdapter<R>) labelToDelegate.get(label);
+        if (delegate == null) {
+          throw new JsonParseException("cannot deserialize " + baseType + " subtype named "
+              + label + "; did you forget to register a subtype?");
+        }
+        return delegate.fromJsonTree(jsonElement);
+      }
+
+      @Override public void write(JsonWriter out, R value) throws IOException {
+        Class<?> srcType = value.getClass();
+        String label = subtypeToLabel.get(srcType);
+        @SuppressWarnings("unchecked") // registration requires that subtype extends T
+            TypeAdapter<R> delegate = (TypeAdapter<R>) subtypeToDelegate.get(srcType);
+        if (delegate == null) {
+          throw new JsonParseException("cannot serialize " + srcType.getName()
+              + "; did you forget to register a subtype?");
+        }
+        JsonObject jsonObject = delegate.toJsonTree(value).getAsJsonObject();
+        if (jsonObject.has(typeFieldName) && !srcType.getSimpleName().equals("OldInput")) {
+          throw new JsonParseException("cannot serialize " + srcType.getName()
+              + " because it already defines a field named " + typeFieldName);
+        }
+        JsonObject clone = new JsonObject();
+        if (!srcType.getSimpleName().equals("OldInput")) {
+          clone.add(typeFieldName, new JsonPrimitive(label));
+        }
+        for (Map.Entry<String, JsonElement> e : jsonObject.entrySet()) {
+          clone.add(e.getKey(), e.getValue());
+        }
+        Streams.write(clone, out);
+      }
+    }.nullSafe();
+  }
+}
+

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/CheckBox.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.display.ui;
+
+import java.awt.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Html Checkbox
+ */
+public class CheckBox extends OptionInput<Object[]> {
+
+  public CheckBox() {
+  }
+
+  public CheckBox(String name, Object[] defaultValue, ParamOption[] options) {
+    this.name = name;
+    this.displayName = name;
+    this.defaultValue = defaultValue;
+    this.options = options;
+  }
+
+  public CheckBox(String name, Collection<Object> defaultValue, ParamOption[] options) {
+    this(name, defaultValue.toArray(), options);
+  }
+
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/OptionInput.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.display.ui;
+
+import org.apache.zeppelin.display.Input;
+
+/**
+ * Base class for Input with options
+ *
+ * @param <T>
+ */
+public abstract class OptionInput<T> extends Input<T> {
+
+  /**
+   * Parameters option.
+   */
+  public static class ParamOption {
+    Object value;
+    String displayName;
+
+    public ParamOption(Object value, String displayName) {
+      super();
+      this.value = value;
+      this.displayName = displayName;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+
+      ParamOption that = (ParamOption) o;
+
+      if (value != null ? !value.equals(that.value) : that.value != null) return false;
+      return displayName != null ? displayName.equals(that.displayName) : that.displayName == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+      int result = value != null ? value.hashCode() : 0;
+      result = 31 * result + (displayName != null ? displayName.hashCode() : 0);
+      return result;
+    }
+
+    public Object getValue() {
+      return value;
+    }
+
+    public void setValue(Object value) {
+      this.value = value;
+    }
+
+    public String getDisplayName() {
+      return displayName;
+    }
+
+    public void setDisplayName(String displayName) {
+      this.displayName = displayName;
+    }
+
+  }
+
+  protected ParamOption[] options;
+
+  public ParamOption[] getOptions() {
+    return options;
+  }
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/Select.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.display.ui;
+
+/**
+ * Html Dropdown list
+ */
+public class Select extends OptionInput<Object> {
+
+  public Select() {
+
+  }
+
+  public Select(String name, Object defaultValue, ParamOption[] options) {
+    this.name = name;
+    this.displayName = name;
+    this.defaultValue = defaultValue;
+    this.options = options;
+  }
+
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/ui/TextBox.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.display.ui;
+
+import org.apache.zeppelin.display.Input;
+
+/**
+ * Html TextBox control
+ */
+public class TextBox extends Input<String> {
+
+  public TextBox() {
+
+  }
+
+  public TextBox(String name, String defaultValue) {
+    this.name = name;
+    this.displayName = name;
+    this.defaultValue = defaultValue;
+  }
+
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java

@@ -152,11 +152,12 @@ public class ClassloaderInterpreter
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
     Thread.currentThread().setContextClassLoader(cl);
     try {
-      List completion = intp.completion(buf, cursor);
+      List completion = intp.completion(buf, cursor, interpreterContext);
       return completion;
     } catch (Exception e) {
       throw new InterpreterException(e);

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -102,10 +102,12 @@ public abstract class Interpreter {
    *
    * @param buf statements
    * @param cursor cursor position in statements
+   * @param interpreterContext
    * @return list of possible completion. Return empty list if there're nothing to return.
    */
   @ZeppelinApi
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext)  {
     return null;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java

@@ -121,9 +121,10 @@ public class LazyOpenInterpreter
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     open();
-    List completion = intp.completion(buf, cursor);
+    List completion = intp.completion(buf, cursor, interpreterContext);
     return completion;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java

@@ -381,14 +381,14 @@ public class RemoteInterpreter extends Interpreter {
       context.getConfig().putAll(remoteConfig);
 
       if (form == FormType.NATIVE) {
-        GUI remoteGui = gson.fromJson(remoteResult.getGui(), GUI.class);
+        GUI remoteGui = GUI.fromJson(remoteResult.getGui());
         currentGUI.clear();
         currentGUI.setParams(remoteGui.getParams());
         currentGUI.setForms(remoteGui.getForms());
       } else if (form == FormType.SIMPLE) {
         final Map<String, Input> currentForms = currentGUI.getForms();
         final Map<String, Object> currentParams = currentGUI.getParams();
-        final GUI remoteGUI = gson.fromJson(remoteResult.getGui(), GUI.class);
+        final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
         final Map<String, Input> remoteForms = remoteGUI.getForms();
         final Map<String, Object> remoteParams = remoteGUI.getParams();
         currentForms.putAll(remoteForms);
@@ -481,7 +481,8 @@ public class RemoteInterpreter extends Interpreter {
 
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
     Client client = null;
     try {
@@ -492,7 +493,8 @@ public class RemoteInterpreter extends Interpreter {
 
     boolean broken = false;
     try {
-      List completion = client.completion(sessionKey, className, buf, cursor);
+      List completion = client.completion(sessionKey, className, buf, cursor,
+          convert(interpreterContext));
       return completion;
     } catch (TException e) {
       broken = true;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -562,10 +562,10 @@ public class RemoteInterpreterServer
 
   @Override
   public List<InterpreterCompletion> completion(String noteId,
-      String className, String buf, int cursor)
+      String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
       throws TException {
     Interpreter intp = getInterpreter(noteId, className);
-    List completion = intp.completion(buf, cursor);
+    List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
     return completion;
   }
 
@@ -592,7 +592,7 @@ public class RemoteInterpreterServer
         gson.fromJson(ric.getAuthenticationInfo(), AuthenticationInfo.class),
         (Map<String, Object>) gson.fromJson(ric.getConfig(),
             new TypeToken<Map<String, Object>>() {}.getType()),
-        gson.fromJson(ric.getGui(), GUI.class),
+        GUI.fromJson(ric.getGui()),
         interpreterGroup.getAngularObjectRegistry(),
         interpreterGroup.getResourcePool(),
         contextRunners, output, remoteWorksController, eventClient);
@@ -737,7 +737,7 @@ public class RemoteInterpreterServer
         result.code().name(),
         msg,
         gson.toJson(config),
-        gson.toJson(gui));
+        gui.toJson());
   }
 
   @Override

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java

@@ -51,12 +51,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class InterpreterCompletion implements org.apache.thrift.TBase<InterpreterCompletion, InterpreterCompletion._Fields>, java.io.Serializable, Cloneable, Comparable<InterpreterCompletion> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InterpreterCompletion");
 
   private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1);
   private static final org.apache.thrift.protocol.TField VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("value", org.apache.thrift.protocol.TType.STRING, (short)2);
+  private static final org.apache.thrift.protocol.TField META_FIELD_DESC = new org.apache.thrift.protocol.TField("meta", org.apache.thrift.protocol.TType.STRING, (short)3);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -66,11 +67,13 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
 
   public String name; // required
   public String value; // required
+  public String meta; // required
 
   /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
   public enum _Fields implements org.apache.thrift.TFieldIdEnum {
     NAME((short)1, "name"),
-    VALUE((short)2, "value");
+    VALUE((short)2, "value"),
+    META((short)3, "meta");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -89,6 +92,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
           return NAME;
         case 2: // VALUE
           return VALUE;
+        case 3: // META
+          return META;
         default:
           return null;
       }
@@ -136,6 +141,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
     tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.DEFAULT, 
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+    tmpMap.put(_Fields.META, new org.apache.thrift.meta_data.FieldMetaData("meta", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(InterpreterCompletion.class, metaDataMap);
   }
@@ -145,11 +152,13 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
 
   public InterpreterCompletion(
     String name,
-    String value)
+    String value,
+    String meta)
   {
     this();
     this.name = name;
     this.value = value;
+    this.meta = meta;
   }
 
   /**
@@ -162,6 +171,9 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     if (other.isSetValue()) {
       this.value = other.value;
     }
+    if (other.isSetMeta()) {
+      this.meta = other.meta;
+    }
   }
 
   public InterpreterCompletion deepCopy() {
@@ -172,6 +184,7 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
   public void clear() {
     this.name = null;
     this.value = null;
+    this.meta = null;
   }
 
   public String getName() {
@@ -222,6 +235,30 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     }
   }
 
+  public String getMeta() {
+    return this.meta;
+  }
+
+  public InterpreterCompletion setMeta(String meta) {
+    this.meta = meta;
+    return this;
+  }
+
+  public void unsetMeta() {
+    this.meta = null;
+  }
+
+  /** Returns true if field meta is set (has been assigned a value) and false otherwise */
+  public boolean isSetMeta() {
+    return this.meta != null;
+  }
+
+  public void setMetaIsSet(boolean value) {
+    if (!value) {
+      this.meta = null;
+    }
+  }
+
   public void setFieldValue(_Fields field, Object value) {
     switch (field) {
     case NAME:
@@ -240,6 +277,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       }
       break;
 
+    case META:
+      if (value == null) {
+        unsetMeta();
+      } else {
+        setMeta((String)value);
+      }
+      break;
+
     }
   }
 
@@ -251,6 +296,9 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     case VALUE:
       return getValue();
 
+    case META:
+      return getMeta();
+
     }
     throw new IllegalStateException();
   }
@@ -266,6 +314,8 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       return isSetName();
     case VALUE:
       return isSetValue();
+    case META:
+      return isSetMeta();
     }
     throw new IllegalStateException();
   }
@@ -301,6 +351,15 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         return false;
     }
 
+    boolean this_present_meta = true && this.isSetMeta();
+    boolean that_present_meta = true && that.isSetMeta();
+    if (this_present_meta || that_present_meta) {
+      if (!(this_present_meta && that_present_meta))
+        return false;
+      if (!this.meta.equals(that.meta))
+        return false;
+    }
+
     return true;
   }
 
@@ -318,6 +377,11 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
     if (present_value)
       list.add(value);
 
+    boolean present_meta = true && (isSetMeta());
+    list.add(present_meta);
+    if (present_meta)
+      list.add(meta);
+
     return list.hashCode();
   }
 
@@ -349,6 +413,16 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         return lastComparison;
       }
     }
+    lastComparison = Boolean.valueOf(isSetMeta()).compareTo(other.isSetMeta());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetMeta()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.meta, other.meta);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
     return 0;
   }
 
@@ -384,6 +458,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       sb.append(this.value);
     }
     first = false;
+    if (!first) sb.append(", ");
+    sb.append("meta:");
+    if (this.meta == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.meta);
+    }
+    first = false;
     sb.append(")");
     return sb.toString();
   }
@@ -443,6 +525,14 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
+          case 3: // META
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.meta = iprot.readString();
+              struct.setMetaIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
           default:
             org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
         }
@@ -468,6 +558,11 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         oprot.writeString(struct.value);
         oprot.writeFieldEnd();
       }
+      if (struct.meta != null) {
+        oprot.writeFieldBegin(META_FIELD_DESC);
+        oprot.writeString(struct.meta);
+        oprot.writeFieldEnd();
+      }
       oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
@@ -492,19 +587,25 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
       if (struct.isSetValue()) {
         optionals.set(1);
       }
-      oprot.writeBitSet(optionals, 2);
+      if (struct.isSetMeta()) {
+        optionals.set(2);
+      }
+      oprot.writeBitSet(optionals, 3);
       if (struct.isSetName()) {
         oprot.writeString(struct.name);
       }
       if (struct.isSetValue()) {
         oprot.writeString(struct.value);
       }
+      if (struct.isSetMeta()) {
+        oprot.writeString(struct.meta);
+      }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, InterpreterCompletion struct) throws org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(2);
+      BitSet incoming = iprot.readBitSet(3);
       if (incoming.get(0)) {
         struct.name = iprot.readString();
         struct.setNameIsSet(true);
@@ -513,6 +614,10 @@ public class InterpreterCompletion implements org.apache.thrift.TBase<Interprete
         struct.value = iprot.readString();
         struct.setValueIsSet(true);
       }
+      if (incoming.get(2)) {
+        struct.meta = iprot.readString();
+        struct.setMetaIsSet(true);
+      }
     }
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteApplicationResult implements org.apache.thrift.TBase<RemoteApplicationResult, RemoteApplicationResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteApplicationResult> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteInterpreterContext, RemoteInterpreterContext._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterContext> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterEvent implements org.apache.thrift.TBase<RemoteInterpreterEvent, RemoteInterpreterEvent._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterEvent> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterResult implements org.apache.thrift.TBase<RemoteInterpreterResult, RemoteInterpreterResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResult> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResultMessage.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterResultMessage implements org.apache.thrift.TBase<RemoteInterpreterResultMessage, RemoteInterpreterResultMessage._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResultMessage> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResultMessage");
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class RemoteInterpreterService {
 
   public interface Iface {
@@ -70,7 +70,7 @@ public class RemoteInterpreterService {
 
     public String getFormType(String sessionKey, String className) throws org.apache.thrift.TException;
 
-    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException;
+    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException;
 
     public void shutdown() throws org.apache.thrift.TException;
 
@@ -126,7 +126,7 @@ public class RemoteInterpreterService {
 
     public void getFormType(String sessionKey, String className, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
-    public void completion(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
+    public void completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
     public void shutdown(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException;
 
@@ -349,19 +349,20 @@ public class RemoteInterpreterService {
       throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getFormType failed: unknown result");
     }
 
-    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException
+    public List<InterpreterCompletion> completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException
     {
-      send_completion(sessionKey, className, buf, cursor);
+      send_completion(sessionKey, className, buf, cursor, interpreterContext);
       return recv_completion();
     }
 
-    public void send_completion(String sessionKey, String className, String buf, int cursor) throws org.apache.thrift.TException
+    public void send_completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext) throws org.apache.thrift.TException
     {
       completion_args args = new completion_args();
       args.setSessionKey(sessionKey);
       args.setClassName(className);
       args.setBuf(buf);
       args.setCursor(cursor);
+      args.setInterpreterContext(interpreterContext);
       sendBase("completion", args);
     }
 
@@ -1064,9 +1065,9 @@ public class RemoteInterpreterService {
       }
     }
 
-    public void completion(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
+    public void completion(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException {
       checkReady();
-      completion_call method_call = new completion_call(sessionKey, className, buf, cursor, resultHandler, this, ___protocolFactory, ___transport);
+      completion_call method_call = new completion_call(sessionKey, className, buf, cursor, interpreterContext, resultHandler, this, ___protocolFactory, ___transport);
       this.___currentMethod = method_call;
       ___manager.call(method_call);
     }
@@ -1076,12 +1077,14 @@ public class RemoteInterpreterService {
       private String className;
       private String buf;
       private int cursor;
-      public completion_call(String sessionKey, String className, String buf, int cursor, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+      private RemoteInterpreterContext interpreterContext;
+      public completion_call(String sessionKey, String className, String buf, int cursor, RemoteInterpreterContext interpreterContext, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
         super(client, protocolFactory, transport, resultHandler, false);
         this.sessionKey = sessionKey;
         this.className = className;
         this.buf = buf;
         this.cursor = cursor;
+        this.interpreterContext = interpreterContext;
       }
 
       public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
@@ -1091,6 +1094,7 @@ public class RemoteInterpreterService {
         args.setClassName(className);
         args.setBuf(buf);
         args.setCursor(cursor);
+        args.setInterpreterContext(interpreterContext);
         args.write(prot);
         prot.writeMessageEnd();
       }
@@ -1933,7 +1937,7 @@ public class RemoteInterpreterService {
 
       public completion_result getResult(I iface, completion_args args) throws org.apache.thrift.TException {
         completion_result result = new completion_result();
-        result.success = iface.completion(args.sessionKey, args.className, args.buf, args.cursor);
+        result.success = iface.completion(args.sessionKey, args.className, args.buf, args.cursor, args.interpreterContext);
         return result;
       }
     }
@@ -2742,7 +2746,7 @@ public class RemoteInterpreterService {
       }
 
       public void start(I iface, completion_args args, org.apache.thrift.async.AsyncMethodCallback<List<InterpreterCompletion>> resultHandler) throws TException {
-        iface.completion(args.sessionKey, args.className, args.buf, args.cursor,resultHandler);
+        iface.completion(args.sessionKey, args.className, args.buf, args.cursor, args.interpreterContext,resultHandler);
       }
     }
 
@@ -9809,6 +9813,7 @@ public class RemoteInterpreterService {
     private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)2);
     private static final org.apache.thrift.protocol.TField BUF_FIELD_DESC = new org.apache.thrift.protocol.TField("buf", org.apache.thrift.protocol.TType.STRING, (short)3);
     private static final org.apache.thrift.protocol.TField CURSOR_FIELD_DESC = new org.apache.thrift.protocol.TField("cursor", org.apache.thrift.protocol.TType.I32, (short)4);
+    private static final org.apache.thrift.protocol.TField INTERPRETER_CONTEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("interpreterContext", org.apache.thrift.protocol.TType.STRUCT, (short)5);
 
     private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
     static {
@@ -9820,13 +9825,15 @@ public class RemoteInterpreterService {
     public String className; // required
     public String buf; // required
     public int cursor; // required
+    public RemoteInterpreterContext interpreterContext; // required
 
     /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
     public enum _Fields implements org.apache.thrift.TFieldIdEnum {
       SESSION_KEY((short)1, "sessionKey"),
       CLASS_NAME((short)2, "className"),
       BUF((short)3, "buf"),
-      CURSOR((short)4, "cursor");
+      CURSOR((short)4, "cursor"),
+      INTERPRETER_CONTEXT((short)5, "interpreterContext");
 
       private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -9849,6 +9856,8 @@ public class RemoteInterpreterService {
             return BUF;
           case 4: // CURSOR
             return CURSOR;
+          case 5: // INTERPRETER_CONTEXT
+            return INTERPRETER_CONTEXT;
           default:
             return null;
         }
@@ -9902,6 +9911,8 @@ public class RemoteInterpreterService {
           new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
       tmpMap.put(_Fields.CURSOR, new org.apache.thrift.meta_data.FieldMetaData("cursor", org.apache.thrift.TFieldRequirementType.DEFAULT, 
           new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+      tmpMap.put(_Fields.INTERPRETER_CONTEXT, new org.apache.thrift.meta_data.FieldMetaData("interpreterContext", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteInterpreterContext.class)));
       metaDataMap = Collections.unmodifiableMap(tmpMap);
       org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(completion_args.class, metaDataMap);
     }
@@ -9913,7 +9924,8 @@ public class RemoteInterpreterService {
       String sessionKey,
       String className,
       String buf,
-      int cursor)
+      int cursor,
+      RemoteInterpreterContext interpreterContext)
     {
       this();
       this.sessionKey = sessionKey;
@@ -9921,6 +9933,7 @@ public class RemoteInterpreterService {
       this.buf = buf;
       this.cursor = cursor;
       setCursorIsSet(true);
+      this.interpreterContext = interpreterContext;
     }
 
     /**
@@ -9938,6 +9951,9 @@ public class RemoteInterpreterService {
         this.buf = other.buf;
       }
       this.cursor = other.cursor;
+      if (other.isSetInterpreterContext()) {
+        this.interpreterContext = new RemoteInterpreterContext(other.interpreterContext);
+      }
     }
 
     public completion_args deepCopy() {
@@ -9951,6 +9967,7 @@ public class RemoteInterpreterService {
       this.buf = null;
       setCursorIsSet(false);
       this.cursor = 0;
+      this.interpreterContext = null;
     }
 
     public String getSessionKey() {
@@ -10048,6 +10065,30 @@ public class RemoteInterpreterService {
       __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __CURSOR_ISSET_ID, value);
     }
 
+    public RemoteInterpreterContext getInterpreterContext() {
+      return this.interpreterContext;
+    }
+
+    public completion_args setInterpreterContext(RemoteInterpreterContext interpreterContext) {
+      this.interpreterContext = interpreterContext;
+      return this;
+    }
+
+    public void unsetInterpreterContext() {
+      this.interpreterContext = null;
+    }
+
+    /** Returns true if field interpreterContext is set (has been assigned a value) and false otherwise */
+    public boolean isSetInterpreterContext() {
+      return this.interpreterContext != null;
+    }
+
+    public void setInterpreterContextIsSet(boolean value) {
+      if (!value) {
+        this.interpreterContext = null;
+      }
+    }
+
     public void setFieldValue(_Fields field, Object value) {
       switch (field) {
       case SESSION_KEY:
@@ -10082,6 +10123,14 @@ public class RemoteInterpreterService {
         }
         break;
 
+      case INTERPRETER_CONTEXT:
+        if (value == null) {
+          unsetInterpreterContext();
+        } else {
+          setInterpreterContext((RemoteInterpreterContext)value);
+        }
+        break;
+
       }
     }
 
@@ -10099,6 +10148,9 @@ public class RemoteInterpreterService {
       case CURSOR:
         return Integer.valueOf(getCursor());
 
+      case INTERPRETER_CONTEXT:
+        return getInterpreterContext();
+
       }
       throw new IllegalStateException();
     }
@@ -10118,6 +10170,8 @@ public class RemoteInterpreterService {
         return isSetBuf();
       case CURSOR:
         return isSetCursor();
+      case INTERPRETER_CONTEXT:
+        return isSetInterpreterContext();
       }
       throw new IllegalStateException();
     }
@@ -10171,6 +10225,15 @@ public class RemoteInterpreterService {
           return false;
       }
 
+      boolean this_present_interpreterContext = true && this.isSetInterpreterContext();
+      boolean that_present_interpreterContext = true && that.isSetInterpreterContext();
+      if (this_present_interpreterContext || that_present_interpreterContext) {
+        if (!(this_present_interpreterContext && that_present_interpreterContext))
+          return false;
+        if (!this.interpreterContext.equals(that.interpreterContext))
+          return false;
+      }
+
       return true;
     }
 
@@ -10198,6 +10261,11 @@ public class RemoteInterpreterService {
       if (present_cursor)
         list.add(cursor);
 
+      boolean present_interpreterContext = true && (isSetInterpreterContext());
+      list.add(present_interpreterContext);
+      if (present_interpreterContext)
+        list.add(interpreterContext);
+
       return list.hashCode();
     }
 
@@ -10249,6 +10317,16 @@ public class RemoteInterpreterService {
           return lastComparison;
         }
       }
+      lastComparison = Boolean.valueOf(isSetInterpreterContext()).compareTo(other.isSetInterpreterContext());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetInterpreterContext()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.interpreterContext, other.interpreterContext);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
       return 0;
     }
 
@@ -10296,6 +10374,14 @@ public class RemoteInterpreterService {
       sb.append("cursor:");
       sb.append(this.cursor);
       first = false;
+      if (!first) sb.append(", ");
+      sb.append("interpreterContext:");
+      if (this.interpreterContext == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.interpreterContext);
+      }
+      first = false;
       sb.append(")");
       return sb.toString();
     }
@@ -10303,6 +10389,9 @@ public class RemoteInterpreterService {
     public void validate() throws org.apache.thrift.TException {
       // check for required fields
       // check for sub-struct validity
+      if (interpreterContext != null) {
+        interpreterContext.validate();
+      }
     }
 
     private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
@@ -10373,6 +10462,15 @@ public class RemoteInterpreterService {
                 org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
               }
               break;
+            case 5: // INTERPRETER_CONTEXT
+              if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+                struct.interpreterContext = new RemoteInterpreterContext();
+                struct.interpreterContext.read(iprot);
+                struct.setInterpreterContextIsSet(true);
+              } else { 
+                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
             default:
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
           }
@@ -10406,6 +10504,11 @@ public class RemoteInterpreterService {
         oprot.writeFieldBegin(CURSOR_FIELD_DESC);
         oprot.writeI32(struct.cursor);
         oprot.writeFieldEnd();
+        if (struct.interpreterContext != null) {
+          oprot.writeFieldBegin(INTERPRETER_CONTEXT_FIELD_DESC);
+          struct.interpreterContext.write(oprot);
+          oprot.writeFieldEnd();
+        }
         oprot.writeFieldStop();
         oprot.writeStructEnd();
       }
@@ -10436,7 +10539,10 @@ public class RemoteInterpreterService {
         if (struct.isSetCursor()) {
           optionals.set(3);
         }
-        oprot.writeBitSet(optionals, 4);
+        if (struct.isSetInterpreterContext()) {
+          optionals.set(4);
+        }
+        oprot.writeBitSet(optionals, 5);
         if (struct.isSetSessionKey()) {
           oprot.writeString(struct.sessionKey);
         }
@@ -10449,12 +10555,15 @@ public class RemoteInterpreterService {
         if (struct.isSetCursor()) {
           oprot.writeI32(struct.cursor);
         }
+        if (struct.isSetInterpreterContext()) {
+          struct.interpreterContext.write(oprot);
+        }
       }
 
       @Override
       public void read(org.apache.thrift.protocol.TProtocol prot, completion_args struct) throws org.apache.thrift.TException {
         TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(4);
+        BitSet incoming = iprot.readBitSet(5);
         if (incoming.get(0)) {
           struct.sessionKey = iprot.readString();
           struct.setSessionKeyIsSet(true);
@@ -10471,6 +10580,11 @@ public class RemoteInterpreterService {
           struct.cursor = iprot.readI32();
           struct.setCursorIsSet(true);
         }
+        if (incoming.get(4)) {
+          struct.interpreterContext = new RemoteInterpreterContext();
+          struct.interpreterContext.read(iprot);
+          struct.setInterpreterContextIsSet(true);
+        }
       }
     }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/ZeppelinServerResourceParagraphRunner.java

@@ -51,7 +51,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-1-25")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2017-3-27")
 public class ZeppelinServerResourceParagraphRunner implements org.apache.thrift.TBase<ZeppelinServerResourceParagraphRunner, ZeppelinServerResourceParagraphRunner._Fields>, java.io.Serializable, Cloneable, Comparable<ZeppelinServerResourceParagraphRunner> {
   private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ZeppelinServerResourceParagraphRunner");
 

zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift

@@ -84,7 +84,8 @@ struct ZeppelinServerResourceParagraphRunner {
  */
 struct InterpreterCompletion {
   1: string name,
-  2: string value
+  2: string value,
+  3: string meta
 }
 
 service RemoteInterpreterService {
@@ -96,7 +97,7 @@ service RemoteInterpreterService {
   void cancel(1: string sessionKey, 2: string className, 3: RemoteInterpreterContext interpreterContext);
   i32 getProgress(1: string sessionKey, 2: string className, 3: RemoteInterpreterContext interpreterContext);
   string getFormType(1: string sessionKey, 2: string className);
-  list<InterpreterCompletion> completion(1: string sessionKey, 2: string className, 3: string buf, 4: i32 cursor);
+  list<InterpreterCompletion> completion(1: string sessionKey, 2: string className, 3: string buf, 4: i32 cursor, 5: RemoteInterpreterContext interpreterContext);
   void shutdown();
 
   string getStatus(1: string sessionKey, 2:string jobId);

zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/GUITest.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.display;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.display.ui.CheckBox;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
+import org.apache.zeppelin.display.ui.Select;
+import org.apache.zeppelin.display.ui.TextBox;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class GUITest {
+
+  private ParamOption[] options = new ParamOption[]{
+      new ParamOption("1", "value_1"),
+      new ParamOption("2", "value_2")
+  };
+
+  private List<Object> checkedItems;
+
+  @Before
+  public void setUp() {
+    checkedItems = new ArrayList<>();
+    checkedItems.add("1");
+  }
+
+  @Test
+  public void testGson() {
+    GUI gui = new GUI();
+    gui.textbox("textbox_1", "default_text_1");
+    gui.select("select_1", "1", options);
+    List<Object> list = new ArrayList();
+    list.add("1");
+    gui.checkbox("checkbox_1", list, options);
+
+    String json = gui.toJson();
+    System.out.println(json);
+    GUI gui2 = GUI.fromJson(json);
+    assertEquals(gui2.toJson(), json);
+    assertEquals(gui2.forms, gui2.forms);
+    assertEquals(gui2.params, gui2.params);
+  }
+
+  // Case 1. Old input forms are created in backend, in this case type is always set
+  @Test
+  public void testOldGson_1() throws IOException {
+
+    GUI gui = new GUI();
+    gui.forms.put("textbox_1", new OldInput.OldTextBox("textbox_1", "default_text_1"));
+    gui.forms.put("select_1", new OldInput.OldSelect("select_1", "1", options));
+    gui.forms.put("checkbox_1",
+        new OldInput.OldCheckBox("checkbox_1", checkedItems, options));
+
+    // convert to old json format.
+    String json = gui.toJson();
+
+    // convert to new input forms
+    GUI gui2 = GUI.fromJson(json);
+    assertTrue(3 == gui2.forms.size());
+    assertTrue(gui2.forms.get("textbox_1") instanceof TextBox);
+    assertEquals("default_text_1", gui2.forms.get("textbox_1").getDefaultValue());
+    assertTrue(gui2.forms.get("select_1") instanceof Select);
+    assertEquals(options, ((Select) gui2.forms.get("select_1")).getOptions());
+    assertTrue(gui2.forms.get("checkbox_1") instanceof CheckBox);
+    assertEquals(options, ((CheckBox) gui2.forms.get("checkbox_1")).getOptions());
+  }
+
+  // Case 2. Old input forms are created in frontend, in this case type is only set for checkbox
+  // Actually this is a bug due to method Input#getInputForm
+  @Test
+  public void testOldGson_2() throws IOException {
+
+    GUI gui = new GUI();
+    gui.forms.put("textbox_1", new OldInput("textbox_1", "default_text_1"));
+    gui.forms.put("select_1", new OldInput("select_1", "1", options));
+    gui.forms.put("checkbox_1",
+        new OldInput.OldCheckBox("checkbox_1", checkedItems, options));
+
+    // convert to old json format.
+    String json = gui.toJson();
+
+    // convert to new input forms
+    GUI gui2 = GUI.fromJson(json);
+    assertTrue(3 == gui2.forms.size());
+    assertTrue(gui2.forms.get("textbox_1") instanceof TextBox);
+    assertEquals("default_text_1", gui2.forms.get("textbox_1").getDefaultValue());
+    assertTrue(gui2.forms.get("select_1") instanceof Select);
+    assertEquals(options, ((Select) gui2.forms.get("select_1")).getOptions());
+    assertTrue(gui2.forms.get("checkbox_1") instanceof CheckBox);
+    assertEquals(options, ((CheckBox) gui2.forms.get("checkbox_1")).getOptions());
+  }
+}

zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java

@@ -20,16 +20,19 @@ package org.apache.zeppelin.display;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.zeppelin.display.ui.CheckBox;
+import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
+import org.apache.zeppelin.display.ui.Select;
+import org.apache.zeppelin.display.ui.TextBox;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNull;
 
-import org.apache.zeppelin.display.Input.ParamOption;
-
 public class InputTest {
 
 	@Before
@@ -42,7 +45,7 @@ public class InputTest {
 
 	@Test
 	public void testFormExtraction() {
-		// input form
+		// textbox form
 		String script = "${input_form=}";
 		Map<String, Input> forms = Input.extractSimpleQueryForm(script);
 		assertEquals(1, forms.size());
@@ -50,50 +53,57 @@ public class InputTest {
 		assertEquals("input_form", form.name);
 		assertNull(form.displayName);
 		assertEquals("", form.defaultValue);
-		assertNull(form.options);
+		assertTrue(form instanceof TextBox);
 
-		// input form with display name & default value
+		// textbox form with display name & default value
 		script = "${input_form(Input Form)=xxx}";
 		forms = Input.extractSimpleQueryForm(script);
 		form = forms.get("input_form");
 		assertEquals("xxx", form.defaultValue);
+		assertTrue(form instanceof TextBox);
 
 		// selection form
 		script = "${select_form(Selection Form)=op1,op1|op2(Option 2)|op3}";
 		form = Input.extractSimpleQueryForm(script).get("select_form");
 		assertEquals("select_form", form.name);
 		assertEquals("op1", form.defaultValue);
+		assertTrue(form instanceof Select);
 		assertArrayEquals(new ParamOption[]{new ParamOption("op1", null),
-				new ParamOption("op2", "Option 2"), new ParamOption("op3", null)}, form.options);
+				new ParamOption("op2", "Option 2"), new ParamOption("op3", null)},
+				((Select) form).getOptions());
 
 		// checkbox form
 		script = "${checkbox:checkbox_form=op1,op1|op2|op3}";
 		form = Input.extractSimpleQueryForm(script).get("checkbox_form");
 		assertEquals("checkbox_form", form.name);
-		assertEquals("checkbox", form.type);
+		assertTrue(form instanceof CheckBox);
+
 		assertArrayEquals(new Object[]{"op1"}, (Object[]) form.defaultValue);
 		assertArrayEquals(new ParamOption[]{new ParamOption("op1", null),
-				new ParamOption("op2", null), new ParamOption("op3", null)}, form.options);
+				new ParamOption("op2", null), new ParamOption("op3", null)},
+				((CheckBox) form).getOptions());
 
 		// checkbox form with multiple default checks
 		script = "${checkbox:checkbox_form(Checkbox Form)=op1|op3,op1(Option 1)|op2|op3}";
 		form = Input.extractSimpleQueryForm(script).get("checkbox_form");
 		assertEquals("checkbox_form", form.name);
 		assertEquals("Checkbox Form", form.displayName);
-		assertEquals("checkbox", form.type);
+		assertTrue(form instanceof CheckBox);
 		assertArrayEquals(new Object[]{"op1", "op3"}, (Object[]) form.defaultValue);
 		assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"),
-				new ParamOption("op2", null), new ParamOption("op3", null)}, form.options);
+				new ParamOption("op2", null), new ParamOption("op3", null)},
+				((CheckBox) form).getOptions());
 
 		// checkbox form with no default check
 		script = "${checkbox:checkbox_form(Checkbox Form)=,op1(Option 1)|op2(Option 2)|op3(Option 3)}";
 		form = Input.extractSimpleQueryForm(script).get("checkbox_form");
 		assertEquals("checkbox_form", form.name);
 		assertEquals("Checkbox Form", form.displayName);
-		assertEquals("checkbox", form.type);
+		assertTrue(form instanceof CheckBox);
 		assertArrayEquals(new Object[]{}, (Object[]) form.defaultValue);
 		assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"),
-				new ParamOption("op2", "Option 2"), new ParamOption("op3", "Option 3")}, form.options);
+				new ParamOption("op2", "Option 2"), new ParamOption("op3", "Option 3")},
+				((CheckBox) form).getOptions());
 	}
 
 
@@ -125,4 +135,5 @@ public class InputTest {
 		assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1\n" +
 				"NEW_CHECKED=nc_a and nc_c", replaced);
 	}
+
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java

@@ -78,7 +78,8 @@ public class MockInterpreterA extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java

@@ -106,7 +106,8 @@ public class MockInterpreterAngular extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java

@@ -78,7 +78,8 @@ public class MockInterpreterB extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }
 

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java

@@ -67,7 +67,8 @@ public class MockInterpreterEnv extends Interpreter {
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String buf, int cursor) {
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+      InterpreterContext interpreterContext) {
     return null;
   }