Merge branch 'master' into ZEPPELIN-2562

.travis.yml

@@ -37,7 +37,7 @@ addons:
 env:
   global:
     # Interpreters does not required by zeppelin-server integration tests
-    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
+    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
 
 matrix:
   include:
@@ -53,7 +53,7 @@ matrix:
       sudo: false
       dist: trusty
       jdk: "oraclejdk8"
-      env: WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
+      env: PYTHON="2" WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
       addons:
         apt:
           sources:
@@ -68,54 +68,54 @@ matrix:
     # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
     - jdk: "oraclejdk8"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
 
     # Test selenium with spark module for 1.6.3
     - jdk: "oraclejdk7"
       dist: precise
-      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+      env: PYTHON="2" TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
 
     # Test interpreter modules
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+      env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="install -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
 
     # Test spark module for 2.2.0 with scala 2.11, livy
     - jdk: "oraclejdk8"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
 
     # Test spark module for 2.1.0 with scala 2.11, livy
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
 
     # Test spark module for 2.0.2 with scala 2.11
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 1.6.3 with scala 2.10
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 1.6.3 with scala 2.11
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test python/pyspark with python 2, livy 0.2
     - sudo: required
       dist: precise
       jdk: "oraclejdk7"
-      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
     # Test python/pyspark with python 3, livy 0.3
     - sudo: required
       dist: precise
       jdk: "oraclejdk7"
-      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
   # check files included in commit range, clear bower_components if a bower.json file has changed.
@@ -141,7 +141,7 @@ install:
 before_script:
   - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
   - if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
-  - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
+  - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-$LIVY_VER-bin; fi
   - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
   - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh
@@ -162,15 +162,13 @@ after_success:
 after_failure:
   - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
   - find . -name rat.txt | xargs cat
+  - cat logs/*
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
   - cat zeppelin-web/npm-debug.log
   - cat spark-*/logs/*
   - cat livy/target/tmp/*/output.log
-  - ls -R livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
-  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
-  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
   - cat livy/target/tmp/livy-int-test/*/output.log
-  - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
-  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
-  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
+  - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stdout
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stderr

alluxio/pom.xml

@@ -47,6 +47,7 @@
         <dependency>
               <groupId>com.google.guava</groupId>
               <artifactId>guava</artifactId>
+              <version>15.0</version>
         </dependency>
 
         <dependency>

beam/README.md

@@ -8,7 +8,7 @@ Current interpreter implementation supports the static repl. It compiles the cod
 You have to first build the Beam interpreter by enable the **beam** profile as follows:
 
 ```
-mvn clean package -Pbeam -DskipTests
+mvn clean package -Pbeam -DskipTests -Pscala-2.10
 ```
 
 ### Notice

beam/pom.xml

@@ -35,7 +35,7 @@
   <properties>
     <beam.hadoop.version>2.3.0</beam.hadoop.version>
     <beam.spark.version>1.6.2</beam.spark.version>
-    <beam.beam.version>0.2.0-incubating</beam.beam.version>
+    <beam.beam.version>2.0.0</beam.beam.version>
 
     <!-- library versions -->
     <netty.version>4.1.1.Final</netty.version>
@@ -211,6 +211,14 @@
       <version>${beam.beam.version}</version>
       <type>jar</type>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-flink_${scala.binary.version}</artifactId>
+      <version>${beam.beam.version}</version>
+      <exclusions>
+      </exclusions>
+    </dependency>
   
     <dependency>
       <groupId>${project.groupId}</groupId>

bin/zeppelin-daemon.sh

@@ -67,6 +67,10 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
   ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
 fi
 
+if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+  ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+fi
+
 # Add jdbc connector jar
 # ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar"
 

bin/zeppelin.sh

@@ -73,6 +73,10 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
 
 ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
 
+if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+  ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+fi
+
 if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
   echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}"
   $(mkdir -p "${ZEPPELIN_LOG_DIR}")

conf/zeppelin-site.xml.template

@@ -173,6 +173,26 @@
 </property>
 -->
 
+<!-- Notebook storage layer using hdfs file system
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
+  <description>hdfs notebook persistence layer implementation</description>
+</property>
+
+<property>
+  <name>zeppelin.hdfs.keytab</name>
+  <value></value>
+  <description>keytab for accessing kerberized hdfs</description>
+</property>
+
+<property>
+  <name>zeppelin.hdfs.principal</name>
+  <value></value>
+  <description>principal for accessing kerberized hdfs</description>
+</property>
+-->
+
 <!-- For connecting your Zeppelin with ZeppelinHub -->
 <!--
 <property>

docs/index.md

@@ -123,11 +123,6 @@ limitations under the License.
   * [Useful Developer Tools](./development/contribution/useful_developer_tools.html)
   * [How to Contribute (code)](./development/contribution/how_to_contribute_code.html)
   * [How to Contribute (website)](./development/contribution/how_to_contribute_website.html)
-
-#### External Resources
-  * [Mailing List](https://zeppelin.apache.org/community.html)
-  * [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
-  * [Stackoverflow Questions about Zeppelin (tag: `apache-zeppelin`)](http://stackoverflow.com/questions/tagged/apache-zeppelin)
   
 #### Available Interpreters 
   * [Alluxio](./interpreter/alluxio.html)
@@ -156,3 +151,7 @@ limitations under the License.
   * [Shell](./interpreter/Shell.html)
   * [Spark](./interpreter/spark.html)
   
+#### External Resources
+  * [Mailing List](https://zeppelin.apache.org/community.html)
+  * [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
+  * [Stackoverflow Questions about Zeppelin (tag: `apache-zeppelin`)](http://stackoverflow.com/questions/tagged/apache-zeppelin)

docs/interpreter/beam.md

@@ -44,18 +44,10 @@ import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.ArrayList;
-import org.apache.spark.api.java.*;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.SparkConf;
-import org.apache.spark.streaming.*;
-import org.apache.spark.SparkContext;
 import org.apache.beam.runners.direct.*;
 import org.apache.beam.sdk.runners.*;
 import org.apache.beam.sdk.options.*;
-import org.apache.beam.runners.spark.*;
-import org.apache.beam.runners.spark.io.ConsoleIO;
 import org.apache.beam.runners.flink.*;
-import org.apache.beam.runners.flink.examples.WordCount.Options;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -89,12 +81,12 @@ public class MinimalWordCount {
 	};  
   static final List<String> SENTENCES = Arrays.asList(SENTENCES_ARRAY);
   public static void main(String[] args) {
-    Options options = PipelineOptionsFactory.create().as(Options.class);
+    PipelineOptions options = PipelineOptionsFactory.create().as(PipelineOptions.class);
     options.setRunner(FlinkRunner.class);
     Pipeline p = Pipeline.create(options);
     p.apply(Create.of(SENTENCES).withCoder(StringUtf8Coder.of()))
          .apply("ExtractWords", ParDo.of(new DoFn<String, String>() {
-           @Override
+           @ProcessElement
            public void processElement(ProcessContext c) {
              for (String word : c.element().split("[^a-zA-Z']+")) {
                if (!word.isEmpty()) {
@@ -105,7 +97,7 @@ public class MinimalWordCount {
          }))
         .apply(Count.<String> perElement())
         .apply("FormatResults", ParDo.of(new DoFn<KV<String, Long>, String>() {
-          @Override
+          @ProcessElement
           public void processElement(DoFn<KV<String, Long>, String>.ProcessContext arg0)
             throws Exception {
             s.add("\n" + arg0.element().getKey() + "\t" + arg0.element().getValue());

docs/interpreter/python.md

@@ -232,6 +232,70 @@ SELECT * FROM rates WHERE age < 40
 Otherwise it can be referred to as `%python.sql`
 
 
+## IPython Support
+
+IPython is more powerful than the default python interpreter with extra functionality. You can use IPython with Python2 or Python3 which depends on which python you set `zeppelin.python`.
+
+   **Pre-requests**
+   
+    - Jupyter `pip install jupyter`
+    - grpcio `pip install grpcio`
+
+If you already install anaconda, then you just need to install `grpcio` as Jupyter is already included in anaconda.
+
+In addition to all basic functions of the python interpreter, you can use all the IPython advanced features as you use it in Jupyter Notebook.
+
+e.g. 
+
+Use IPython magic
+
+```
+%python.ipython
+
+#python help
+range?
+
+#timeit
+%timeit range(100)
+```
+
+Use matplotlib 
+
+```
+%python.ipython
+
+
+%matplotlib inline
+import matplotlib.pyplot as plt
+
+print("hello world")
+data=[1,2,3,4]
+plt.figure()
+plt.plot(data)
+```
+
+We also make `ZeppelinContext` available in IPython Interpreter. You can use `ZeppelinContext` to create dynamic forms and display pandas DataFrame.
+
+e.g.
+
+Create dynamic form
+
+```
+z.input(name='my_name', defaultValue='hello')
+```
+
+Show pandas dataframe
+
+```
+import pandas as pd
+df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})
+z.show(df)
+
+```
+
+By default, we would use IPython in `%python.python` if IPython is available. Otherwise it would fall back to the original Python implementation.
+If you don't want to use IPython, then you can set `zeppelin.python.useIPython` as `false` in interpreter setting.
+
 ## Technical description
 
 For in-depth technical details on current implementation please refer to [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md).

docs/interpreter/shell.md

@@ -43,6 +43,11 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
     <td>60000</td>
     <td>Shell command time out in millisecs</td>
   </tr>
+  <tr>
+    <td>shell.working.directory.user.home</td>
+    <td>false</td>
+    <td>If this set to true, the shell's working directory will be set to user home</td>
+  </tr>
   <tr>
     <td>zeppelin.shell.auth.type</td>
     <td></td>

docs/interpreter/spark.md

@@ -414,6 +414,12 @@ You can choose one of `shared`, `scoped` and `isolated` options wheh you configu
 Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). 
 It creates separated SparkContext per each notebook in `isolated` mode.
 
+## IPython support
+
+By default, zeppelin would use IPython in `pyspark` when IPython is available, Otherwise it would fall back to the original PySpark implementation.
+If you don't want to use IPython, then you can set `zeppelin.spark.useIPython` as `false` in interpreter setting. For the IPython features, you can refer doc
+[Python Interpreter](python.html)
+
 
 ## Setting up Zeppelin with Kerberos
 Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark on YARN:

docs/setup/security/notebook_authorization.md

@@ -36,13 +36,17 @@ As you can see, each Zeppelin notebooks has 3 entities :
 * Owners ( users or groups )
 * Readers ( users or groups )
 * Writers ( users or groups )
+* Runners ( users or groups )
 
 <center><img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/permission_setting.png"></center>
 
 Fill out the each forms with comma seperated **users** and **groups** configured in `conf/shiro.ini` file.
 If the form is empty (*), it means that any users can perform that operation.
 
-If someone who doesn't have **read** permission is trying to access the notebook or someone who doesn't have **write** permission is trying to edit the notebook, Zeppelin will ask to login or block the user.
+If someone who doesn't have **read** permission is trying to access the notebook or someone who doesn't have **write** permission is trying to edit the notebook,
+or someone who doesn't have **run** permission is trying to run a paragraph Zeppelin will ask to login or block the user.
+
+By default, owners and writers have **write** permission, owners, writers and runners have **run** permission, owners, writers, runners and readers have **read** permission
 
 <center><img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/insufficient_privileges.png"></center>
 
@@ -63,13 +67,13 @@ or set `zeppelin.notebook.public` property to `false` in `conf/zeppelin-site.xml
 </property>
 ```
 
-Behind the scenes, when you create a new note only the `owners` field is filled with current user, leaving `readers` and `writers` fields empty. All the notes with at least one empty authorization field are considered to be in `public` workspace. Thus when setting `zeppelin.notebook.public` (or corresponding `ZEPPELIN_NOTEBOOK_PUBLIC`) to false, newly created notes have `readers` and `writers` fields filled with current user, making note appear as in `private` workspace.
+Behind the scenes, when you create a new note only the `owners` field is filled with current user, leaving `readers`, `runners` and `writers` fields empty. All the notes with at least one empty authorization field are considered to be in `public` workspace. Thus when setting `zeppelin.notebook.public` (or corresponding `ZEPPELIN_NOTEBOOK_PUBLIC`) to false, newly created notes have `readers`, `runners`, `writers` fields filled with current user, making note appear as in `private` workspace.
 
 ## How it works
 In this section, we will explain the detail about how the notebook authorization works in backend side.
 
 ### NotebookServer
-The [NotebookServer](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java) classifies every notebook operations into three categories: **Read**, **Write**, **Manage**.
+The [NotebookServer](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java) classifies every notebook operations into three categories: **Read**, **Run**, **Write**, **Manage**.
 Before executing a notebook operation, it checks if the user and the groups associated with the `NotebookSocket` have permissions.
 For example, before executing a **Read** operation, it checks if the user and the groups have at least one entity that belongs to the **Reader** entities.
 

docs/setup/security/shiro_authentication.md

@@ -210,6 +210,21 @@ securityManager.realms = $zeppelinHubRealm
 
 > Note: ZeppelinHub is not releated to Apache Zeppelin project.
 
+## Secure Cookie for Zeppelin Sessions (optional)
+Zeppelin can be configured to set `HttpOnly` flag in the session cookie. With this configuration, Zeppelin cookies can 
+not be accessed via client side scripts thus preventing majority of Cross-site scripting (XSS) attacks.
+
+To enable secure cookie support via Shiro, add the following lines in `conf/shiro.ini` under `[main]` section, after
+defining a `sessionManager`.
+
+```
+cookie = org.apache.shiro.web.servlet.SimpleCookie
+cookie.name = JSESSIONID
+cookie.secure = true
+cookie.httpOnly = true
+sessionManager.sessionIdCookie = $cookie
+```
+
 ## Secure your Zeppelin information (optional)
 By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
 Sometimes you might want to hide these information for your use case.

docs/setup/storage/storage.md

@@ -30,6 +30,7 @@ There are few notebook storage systems available for a use out of the box:
 
   * (default) use local file system and version it using local Git repository - `GitNotebookRepo`
   * all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
+  * all notes are saved in the notebook folder in hdfs - `HdfsNotebookRepo`
   * storage using Amazon S3 service - `S3NotebookRepo`
   * storage using Azure service - `AzureNotebookRepo`
   * storage using MongoDB - `MongoNotebookRepo`
@@ -51,6 +52,22 @@ To enable versioning for all your local notebooks though a standard Git reposito
 </property>
 ```
 
+</br>
+
+## Notebook Storage in Hdfs repository <a name="Hdfs"></a>
+
+Notes may be stored in hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `HdfsNotebookRepo`, then `zeppelin.notebook.dir` is the path on hdfs. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files.
+If your hadoop cluster is kerberized, then you need to specify `zeppelin.hdfs.keytab` and `zeppelin.hdfs.principal`
+
+```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
+  <description>hdfs notebook persistence layer implementation</description>
+</property>
+```
+
+
 </br>
 
 ## Notebook Storage in S3 <a name="S3"></a>

docs/usage/rest_api/notebook.md

@@ -1215,6 +1215,9 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
       "owners":[  
          "user1"
       ],
+      "runners":[
+         "user2"
+      ],
       "writers":[  
          "user2"
       ]
@@ -1259,6 +1262,9 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
   "owners": [
     "user2"
   ],
+  "runners":[
+    "user2"
+  ],
   "writers": [
     "user1"
   ]

helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java

@@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
   }
 
   @Override
-  protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+  protected Interpreter getInterpreter(String sessionId, String className) throws TException {
     synchronized (this) {
       InterpreterGroup interpreterGroup = getInterpreterGroup();
       if (interpreterGroup == null || interpreterGroup.isEmpty()) {
         createInterpreter(
             "dev",
-            sessionKey,
+            sessionId,
             DevInterpreter.class.getName(),
             new HashMap<String, String>(),
             "anonymous");
@@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
       }
     }
 
-    Interpreter intp = super.getInterpreter(sessionKey, className);
+    Interpreter intp = super.getInterpreter(sessionId, className);
     interpreter = (DevInterpreter) (
         ((LazyOpenInterpreter) intp).getInnerInterpreter());
     interpreter.setInterpreterEvent(this);
-    return super.getInterpreter(sessionKey, className);
+    return super.getInterpreter(sessionId, className);
   }
 
   @Override

ignite/pom.xml

@@ -32,7 +32,7 @@
   <name>Zeppelin: Apache Ignite interpreter</name>
 
   <properties>
-    <ignite.version>1.9.0</ignite.version>
+    <ignite.version>2.1.0</ignite.version>
   </properties>
 
   <dependencies>

ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java

@@ -24,7 +24,6 @@ import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.zeppelin.interpreter.InterpreterContext;

jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java

@@ -179,7 +179,8 @@ public class SqlCompleter {
 
 
   private static void fillTableNames(String schema, DatabaseMetaData meta, Set<String> tables) {
-    try (ResultSet tbls = meta.getTables(schema, schema, "%", null)) {
+    try (ResultSet tbls = meta.getTables(schema, schema, "%",
+        new String[]{"TABLE", "VIEW", "ALIAS", "SYNONYM", "GLOBAL TEMPORARY", "LOCAL TEMPORARY"})) {
       while (tbls.next()) {
         String table = tbls.getString("TABLE_NAME");
         tables.add(table);

livy/pom.xml

@@ -19,544 +19,291 @@
 <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zeppelin</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.8.0-SNAPSHOT</version>
+        <relativePath>..</relativePath>
+    </parent>
 
-  <parent>
-    <artifactId>zeppelin</artifactId>
     <groupId>org.apache.zeppelin</groupId>
+    <artifactId>zeppelin-livy</artifactId>
+    <packaging>jar</packaging>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
-  </parent>
+    <name>Zeppelin: Livy interpreter</name>
 
-  <groupId>org.apache.zeppelin</groupId>
-  <artifactId>zeppelin-livy</artifactId>
-  <packaging>jar</packaging>
-  <version>0.8.0-SNAPSHOT</version>
-  <name>Zeppelin: Livy interpreter</name>
+    <properties>
+        <!--library versions-->
+        <commons.exec.version>1.3</commons.exec.version>
+        <spring.web.version>4.3.0.RELEASE</spring.web.version>
+        <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
 
-  <properties>
-    <!--library versions-->
-    <commons.exec.version>1.3</commons.exec.version>
-    <spring.web.version>4.3.0.RELEASE</spring.web.version>
-    <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
-
-    <!--test library versions-->
-    <livy.version>0.3.0</livy.version>
-    <spark.version>2.1.0</spark.version>
-    <hadoop.version>2.6.0</hadoop.version>
-    <!--plugin versions-->
-    <plugin.failsafe.version>2.16</plugin.failsafe.version>
-    <plugin.antrun.version>1.8</plugin.antrun.version>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>zeppelin-interpreter</artifactId>
-      <version>${project.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-exec</artifactId>
-      <version>${commons.exec.version}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.code.gson</groupId>
-      <artifactId>gson</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.springframework.security.kerberos</groupId>
-      <artifactId>spring-security-kerberos-client</artifactId>
-      <version>${spring.security.kerberosclient}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>org.springframework</groupId>
-      <artifactId>spring-web</artifactId>
-      <version>${spring.web.version}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.cloudera.livy</groupId>
-      <artifactId>livy-integration-test</artifactId>
-      <version>${livy.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.xerial.snappy</groupId>
-          <artifactId>snappy-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-core_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-repl_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-auth</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-hdfs</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-client</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-client</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-tests</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>com.cloudera.livy</groupId>
-      <artifactId>livy-test-lib</artifactId>
-      <version>${livy.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.xerial.snappy</groupId>
-          <artifactId>snappy-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-core_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-repl_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.esotericsoftware</groupId>
-          <artifactId>kryo-shaded</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-repl_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-web-proxy</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-client</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-api</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-tests</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-
-  <repositories>
-    <repository>
-      <id>ossrh</id>
-      <name>ossrh repository</name>
-      <url>https://oss.sonatype.org/content/repositories/releases/</url>
-      <releases>
-        <enabled>true</enabled>
-      </releases>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-    </repository>
-  </repositories>
-
-  <build>
-    <plugins>
-      <plugin>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/livy
-              </outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/livy
-              </outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-failsafe-plugin</artifactId>
-        <version>${plugin.failsafe.version}</version>
-        <executions>
-          <execution>
-            <goals>
-              <goal>integration-test</goal>
-              <goal>verify</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <systemPropertyVariables>
-            <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
-          </systemPropertyVariables>
-          <environmentVariables>
-            <LIVY_SPARK_SCALA_VERSION>${scala.binary.version}</LIVY_SPARK_SCALA_VERSION>
-            <LIVY_LOG_DIR>${project.build.directory}/tmp</LIVY_LOG_DIR>
-          </environmentVariables>
-          <argLine>-Xmx2048m</argLine>
-        </configuration>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <version>${plugin.antrun.version}</version>
-        <executions>
-          <!-- Cleans up files that tests append to (because we have two test plugins). -->
-          <execution>
-            <id>pre-test-clean</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <configuration>
-              <target>
-                <delete file="${project.build.directory}/unit-tests.log"
-                        quiet="true"/>
-                <delete file="${project.build.directory}/jacoco.exec" quiet="true"/>
-                <delete dir="${project.build.directory}/tmp" quiet="true"/>
-              </target>
-            </configuration>
-          </execution>
-          <!-- Create the temp directory to be  used by tests. -->
-          <execution>
-            <id>create-tmp-dir</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <configuration>
-              <target>
-                <mkdir dir="${project.build.directory}/tmp"/>
-              </target>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-
-  <profiles>
-    <profile>
-      <id>livy-0.3</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <livy.version>0.3.0</livy.version>
+        <!--test library versions-->
+        <livy.version>0.4.0-incubating</livy.version>
         <spark.version>2.1.0</spark.version>
-        <hadoop.version>2.6.0</hadoop.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>com.cloudera.livy</groupId>
-          <artifactId>livy-core_${scala.binary.version}</artifactId>
-          <version>0.3.0</version>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.xerial.snappy</groupId>
-              <artifactId>snappy-java</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-core_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
-    </profile>
+        <!--plugin versions-->
+        <plugin.failsafe.version>2.16</plugin.failsafe.version>
+        <plugin.antrun.version>1.8</plugin.antrun.version>
+    </properties>
 
-    <profile>
-      <id>livy-0.2</id>
-      <properties>
-        <livy.version>0.2.0</livy.version>
-        <spark.version>1.6.2</spark.version>
-        <hadoop.version>2.6.0</hadoop.version>
-        <scala.binary.version>2.10</scala.binary.version>
-      </properties>
-      <dependencies>
+    <dependencies>
         <dependency>
-          <groupId>com.cloudera.livy</groupId>
-          <artifactId>livy-core</artifactId>
-          <version>0.2.0</version>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.xerial.snappy</groupId>
-              <artifactId>snappy-java</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-core_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-            </exclusion>
-          </exclusions>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>zeppelin-interpreter</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-compiler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-library</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-reflect</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
+
+        <dependency>
+            <groupId>org.apache.livy</groupId>
+            <artifactId>livy-integration-test</artifactId>
+            <version>${livy.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-core_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-sql_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-hive_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-repl_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.livy</groupId>
+            <artifactId>livy-test-lib</artifactId>
+            <version>${livy.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-core_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-sql_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-hive_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-repl_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-exec</artifactId>
+            <version>${commons.exec.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.security.kerberos</groupId>
+            <artifactId>spring-security-kerberos-client</artifactId>
+            <version>${spring.security.kerberosclient}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-web</artifactId>
+            <version>${spring.web.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>enforce</id>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/../../interpreter/livy
+                            </outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                            <includeScope>runtime</includeScope>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>copy-artifact</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/../../interpreter/livy
+                            </outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                            <includeScope>runtime</includeScope>
+                            <artifactItems>
+                                <artifactItem>
+                                    <groupId>${project.groupId}</groupId>
+                                    <artifactId>${project.artifactId}</artifactId>
+                                    <version>${project.version}</version>
+                                    <type>${project.packaging}</type>
+                                </artifactItem>
+                            </artifactItems>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <version>${plugin.failsafe.version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <systemPropertyVariables>
+                        <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
+                    </systemPropertyVariables>
+                    <environmentVariables>
+                        <LIVY_SPARK_SCALA_VERSION>${scala.binary.version}</LIVY_SPARK_SCALA_VERSION>
+                        <LIVY_LOG_DIR>${project.build.directory}/tmp</LIVY_LOG_DIR>
+                    </environmentVariables>
+                    <argLine>-Xmx2048m</argLine>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>${plugin.antrun.version}</version>
+                <executions>
+                    <!-- Cleans up files that tests append to (because we have two test plugins). -->
+                    <execution>
+                        <id>pre-test-clean</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <delete file="${project.build.directory}/unit-tests.log"
+                                        quiet="true"/>
+                                <delete file="${project.build.directory}/jacoco.exec" quiet="true"/>
+                                <delete dir="${project.build.directory}/tmp" quiet="true"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                    <!-- Create the temp directory to be  used by tests. -->
+                    <execution>
+                        <id>create-tmp-dir</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <mkdir dir="${project.build.directory}/tmp"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>

livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java

@@ -18,8 +18,8 @@
 package org.apache.zeppelin.livy;
 
 
-import com.cloudera.livy.test.framework.Cluster;
-import com.cloudera.livy.test.framework.Cluster$;
+import org.apache.livy.test.framework.Cluster;
+import org.apache.livy.test.framework.Cluster$;
 import org.apache.commons.io.IOUtils;
 import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.user.AuthenticationInfo;
@@ -74,7 +74,7 @@ public class LivyInterpreterIT {
     return true;
   }
 
-  @Test
+//  @Test
   public void testSparkInterpreterRDD() {
     if (!checkPreCondition()) {
       return;
@@ -194,7 +194,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testSparkInterpreterDataFrame() {
     if (!checkPreCondition()) {
       return;
@@ -282,7 +282,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testSparkSQLInterpreter() {
     if (!checkPreCondition()) {
       return;
@@ -317,7 +317,7 @@ public class LivyInterpreterIT {
   }
 
 
-  @Test
+//  @Test
   public void testSparkSQLCancellation() {
     if (!checkPreCondition()) {
       return;
@@ -398,7 +398,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testStringWithTruncation() {
     if (!checkPreCondition()) {
       return;
@@ -458,7 +458,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testStringWithoutTruncation() {
     if (!checkPreCondition()) {
       return;
@@ -633,7 +633,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testSparkInterpreterWithDisplayAppInfo() {
     if (!checkPreCondition()) {
       return;
@@ -672,7 +672,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testSparkRInterpreter() throws LivyException {
     if (!checkPreCondition()) {
       return;
@@ -744,7 +744,7 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
+//  @Test
   public void testLivyTutorialNote() throws IOException {
     if (!checkPreCondition()) {
       return;

pom.xml

@@ -101,7 +101,6 @@
     <libthrift.version>0.9.2</libthrift.version>
     <gson.version>2.2</gson.version>
     <gson-extras.version>0.2.1</gson-extras.version>
-    <guava.version>15.0</guava.version>
     <jetty.version>9.2.15.v20160210</jetty.version>
     <httpcomponents.core.version>4.4.1</httpcomponents.core.version>
     <httpcomponents.client.version>4.5.1</httpcomponents.client.version>
@@ -246,12 +245,6 @@
         <version>${commons.cli.version}</version>
       </dependency>
 
-      <dependency>
-        <groupId>com.google.guava</groupId>
-        <artifactId>guava</artifactId>
-        <version>${guava.version}</version>
-      </dependency>
-
       <!-- Apache Shiro -->
       <dependency>
         <groupId>org.apache.shiro</groupId>
@@ -404,7 +397,7 @@
             </goals>
             <configuration>
               <failOnViolation>true</failOnViolation>
-              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*</excludes>
+              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
             </configuration>
           </execution>
           <execution>
@@ -414,7 +407,7 @@
               <goal>checkstyle-aggregate</goal>
             </goals>
             <configuration>
-              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*</excludes>
+              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
             </configuration>
           </execution>
         </executions>
@@ -1082,6 +1075,8 @@
                <!--The following files are mechanical-->
               <exclude>**/R/rzeppelin/DESCRIPTION</exclude>
               <exclude>**/R/rzeppelin/NAMESPACE</exclude>
+
+              <exclude>python/src/main/resources/grpc/**/*</exclude>
             </excludes>
           </configuration>
 

python/README.md

@@ -50,3 +50,22 @@ mvn -Dpython.test.exclude='' test -pl python -am
  * Matplotlib figures are displayed inline with the notebook automatically using a built-in backend for zeppelin in conjunction with a post-execute hook.
 
  * `%python.sql` support for Pandas DataFrames is optional and provided using https://github.com/yhat/pandasql if user have one installed
+
+
+# IPython Overview
+IPython interpreter for Apache Zeppelin
+
+# IPython Requirements
+You need to install the following python packages to make the IPython interpreter work.
+ * jupyter 5.x
+ * IPython
+ * ipykernel
+ * grpcio
+ 
+If you have installed anaconda, then you just need to install grpc.
+
+# IPython Architecture
+Current interpreter delegate the whole work to ipython kernel via `jupyter_client`. Zeppelin would launch a python process which host the ipython kernel.
+Zeppelin interpreter process will communicate with the python process via `grpc`. Ideally every feature works in IPython should work in Zeppelin as well.
+
+

python/pom.xml

@@ -41,6 +41,7 @@
     </python.test.exclude>
     <pypi.repo.url>https://pypi.python.org/packages</pypi.repo.url>
     <python.py4j.repo.folder>/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</python.py4j.repo.folder>
+    <grpc.version>1.4.0</grpc.version>
   </properties>
 
   <dependencies>
@@ -73,6 +74,28 @@
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-protobuf</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-stub</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>18.0</version>
+    </dependency>
+
     <!-- test libraries -->
     <dependency>
       <groupId>junit</groupId>
@@ -88,7 +111,36 @@
   </dependencies>
 
   <build>
+
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.4.1.Final</version>
+      </extension>
+    </extensions>
+
     <plugins>
+
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.5.0</version>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}</protocArtifact>
+          <pluginId>grpc-java</pluginId>
+          <pluginArtifact>io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}</pluginArtifact>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>compile-custom</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
         <version>1.3.1</version>
@@ -136,6 +188,19 @@
         </executions>
       </plugin>
 
+      <!-- publish test jar as well so that spark module can use it -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>3.0.2</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
 
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

python/src/main/java/org/apache/zeppelin/python/IPythonClient.java

@@ -0,0 +1,211 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.stub.StreamObserver;
+import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
+import org.apache.zeppelin.python.proto.CancelRequest;
+import org.apache.zeppelin.python.proto.CancelResponse;
+import org.apache.zeppelin.python.proto.CompletionRequest;
+import org.apache.zeppelin.python.proto.CompletionResponse;
+import org.apache.zeppelin.python.proto.ExecuteRequest;
+import org.apache.zeppelin.python.proto.ExecuteResponse;
+import org.apache.zeppelin.python.proto.ExecuteStatus;
+import org.apache.zeppelin.python.proto.IPythonGrpc;
+import org.apache.zeppelin.python.proto.OutputType;
+import org.apache.zeppelin.python.proto.StatusRequest;
+import org.apache.zeppelin.python.proto.StatusResponse;
+import org.apache.zeppelin.python.proto.StopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Grpc client for IPython kernel
+ */
+public class IPythonClient {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonClient.class.getName());
+
+  private final ManagedChannel channel;
+  private final IPythonGrpc.IPythonBlockingStub blockingStub;
+  private final IPythonGrpc.IPythonStub asyncStub;
+
+  private Random random = new Random();
+
+  /**
+   * Construct client for accessing RouteGuide server at {@code host:port}.
+   */
+  public IPythonClient(String host, int port) {
+    this(ManagedChannelBuilder.forAddress(host, port).usePlaintext(true));
+  }
+
+  /**
+   * Construct client for accessing RouteGuide server using the existing channel.
+   */
+  public IPythonClient(ManagedChannelBuilder<?> channelBuilder) {
+    channel = channelBuilder.build();
+    blockingStub = IPythonGrpc.newBlockingStub(channel);
+    asyncStub = IPythonGrpc.newStub(channel);
+  }
+
+  public void shutdown() throws InterruptedException {
+    channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
+  }
+
+  // execute the code and make the output as streaming by writing it to InterpreterOutputStream
+  // one by one.
+  public ExecuteResponse stream_execute(ExecuteRequest request,
+                                        final InterpreterOutputStream interpreterOutput) {
+    final ExecuteResponse.Builder finalResponseBuilder = ExecuteResponse.newBuilder()
+        .setStatus(ExecuteStatus.SUCCESS);
+    final AtomicBoolean completedFlag = new AtomicBoolean(false);
+    LOGGER.debug("stream_execute code:\n" + request.getCode());
+    asyncStub.execute(request, new StreamObserver<ExecuteResponse>() {
+      int index = 0;
+      boolean isPreviousOutputImage = false;
+
+      @Override
+      public void onNext(ExecuteResponse executeResponse) {
+        if (executeResponse.getType() == OutputType.TEXT) {
+          try {
+            LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput());
+            if (isPreviousOutputImage) {
+              // add '\n' when switch from image to text
+              interpreterOutput.write("\n".getBytes());
+            }
+            isPreviousOutputImage = false;
+            interpreterOutput.write(executeResponse.getOutput().getBytes());
+            interpreterOutput.getInterpreterOutput().flush();
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+        }
+        if (executeResponse.getType() == OutputType.IMAGE) {
+          try {
+            LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA");
+            if (index != 0) {
+              // add '\n' if this is the not the first element. otherwise it would mix the image
+              // with the text
+              interpreterOutput.write("\n".getBytes());
+            }
+            interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes());
+            interpreterOutput.getInterpreterOutput().flush();
+            isPreviousOutputImage = true;
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+        }
+        if (executeResponse.getStatus() == ExecuteStatus.ERROR) {
+          // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse would
+          // be SUCCESS.
+          finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
+        }
+        index++;
+      }
+
+      @Override
+      public void onError(Throwable throwable) {
+        try {
+          interpreterOutput.getInterpreterOutput().flush();
+        } catch (IOException e) {
+          LOGGER.error("Unexpected IOException", e);
+        }
+        LOGGER.error("Fail to call IPython grpc", throwable);
+      }
+
+      @Override
+      public void onCompleted() {
+        synchronized (completedFlag) {
+          try {
+            LOGGER.debug("stream_execute is completed");
+            interpreterOutput.getInterpreterOutput().flush();
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+          completedFlag.set(true);
+          completedFlag.notify();
+        }
+      }
+    });
+
+    synchronized (completedFlag) {
+      if (!completedFlag.get()) {
+        try {
+          completedFlag.wait();
+        } catch (InterruptedException e) {
+          LOGGER.error("Unexpected Interruption", e);
+        }
+      }
+    }
+    return finalResponseBuilder.build();
+  }
+
+  // blocking execute the code
+  public ExecuteResponse block_execute(ExecuteRequest request) {
+    ExecuteResponse.Builder responseBuilder = ExecuteResponse.newBuilder();
+    responseBuilder.setStatus(ExecuteStatus.SUCCESS);
+    Iterator<ExecuteResponse> iter = blockingStub.execute(request);
+    StringBuilder outputBuilder = new StringBuilder();
+    while (iter.hasNext()) {
+      ExecuteResponse nextResponse = iter.next();
+      if (nextResponse.getStatus() == ExecuteStatus.ERROR) {
+        responseBuilder.setStatus(ExecuteStatus.ERROR);
+      }
+      outputBuilder.append(nextResponse.getOutput());
+    }
+    responseBuilder.setOutput(outputBuilder.toString());
+    return responseBuilder.build();
+  }
+
+  public CancelResponse cancel(CancelRequest request) {
+    return blockingStub.cancel(request);
+  }
+
+  public CompletionResponse complete(CompletionRequest request) {
+    return blockingStub.complete(request);
+  }
+
+  public StatusResponse status(StatusRequest request) {
+    return blockingStub.status(request);
+  }
+
+  public void stop(StopRequest request) {
+    asyncStub.stop(request, null);
+  }
+
+
+  public static void main(String[] args) {
+    IPythonClient client = new IPythonClient("localhost", 50053);
+    client.status(StatusRequest.newBuilder().build());
+
+    ExecuteResponse response = client.block_execute(ExecuteRequest.newBuilder().
+        setCode("abcd=2").build());
+    System.out.println(response.getOutput());
+
+  }
+}

python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java

@@ -0,0 +1,359 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.exec.ExecuteResultHandler;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.LogOutputStream;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
+import org.apache.zeppelin.python.proto.CancelRequest;
+import org.apache.zeppelin.python.proto.CompletionRequest;
+import org.apache.zeppelin.python.proto.CompletionResponse;
+import org.apache.zeppelin.python.proto.ExecuteRequest;
+import org.apache.zeppelin.python.proto.ExecuteResponse;
+import org.apache.zeppelin.python.proto.ExecuteStatus;
+import org.apache.zeppelin.python.proto.IPythonStatus;
+import org.apache.zeppelin.python.proto.StatusRequest;
+import org.apache.zeppelin.python.proto.StatusResponse;
+import org.apache.zeppelin.python.proto.StopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import py4j.GatewayServer;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * IPython Interpreter for Zeppelin
+ */
+public class IPythonInterpreter extends Interpreter implements ExecuteResultHandler {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreter.class);
+
+  private ExecuteWatchdog watchDog;
+  private IPythonClient ipythonClient;
+  private GatewayServer gatewayServer;
+
+  private PythonZeppelinContext zeppelinContext;
+  private String pythonExecutable;
+  private long ipythonLaunchTimeout;
+  private String additionalPythonPath;
+  private String additionalPythonInitFile;
+
+  private InterpreterOutputStream interpreterOutput = new InterpreterOutputStream(LOGGER);
+
+  public IPythonInterpreter(Properties properties) {
+    super(properties);
+  }
+
+  /**
+   * Sub class can customize the interpreter by adding more python packages under PYTHONPATH.
+   * e.g. PySparkInterpreter
+   *
+   * @param additionalPythonPath
+   */
+  public void setAdditionalPythonPath(String additionalPythonPath) {
+    this.additionalPythonPath = additionalPythonPath;
+  }
+
+  /**
+   * Sub class can customize the interpreter by running additional python init code.
+   * e.g. PySparkInterpreter
+   *
+   * @param additionalPythonInitFile
+   */
+  public void setAdditionalPythonInitFile(String additionalPythonInitFile) {
+    this.additionalPythonInitFile = additionalPythonInitFile;
+  }
+
+  @Override
+  public void open() {
+    try {
+      if (ipythonClient != null) {
+        // IPythonInterpreter might already been opened by PythonInterpreter
+        return;
+      }
+      pythonExecutable = getProperty().getProperty("zeppelin.python", "python");
+      ipythonLaunchTimeout = Long.parseLong(
+          getProperty().getProperty("zeppelin.ipython.launch.timeout", "30000"));
+      this.zeppelinContext = new PythonZeppelinContext(
+          getInterpreterGroup().getInterpreterHookRegistry(),
+          Integer.parseInt(getProperty().getProperty("zeppelin.python.maxResult", "1000")));
+      int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+      int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+      LOGGER.info("Launching IPython Kernel at port: " + ipythonPort);
+      LOGGER.info("Launching JVM Gateway at port: " + jvmGatewayPort);
+      ipythonClient = new IPythonClient("127.0.0.1", ipythonPort);
+      launchIPythonKernel(ipythonPort);
+      setupJVMGateway(jvmGatewayPort);
+    } catch (Exception e) {
+      throw new RuntimeException("Fail to open IPythonInterpreter", e);
+    }
+  }
+
+  public boolean checkIPythonPrerequisite() {
+    ProcessBuilder processBuilder = new ProcessBuilder("pip", "freeze");
+    try {
+      File stderrFile = File.createTempFile("zeppelin", ".txt");
+      processBuilder.redirectError(stderrFile);
+      File stdoutFile = File.createTempFile("zeppelin", ".txt");
+      processBuilder.redirectOutput(stdoutFile);
+
+      Process proc = processBuilder.start();
+      int ret = proc.waitFor();
+      if (ret != 0) {
+        LOGGER.warn("Fail to run pip freeze.\n" +
+            IOUtils.toString(new FileInputStream(stderrFile)));
+        return false;
+      }
+      String freezeOutput = IOUtils.toString(new FileInputStream(stdoutFile));
+      if (!freezeOutput.contains("jupyter-client=")) {
+        InterpreterContext.get().out.write("jupyter-client is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("ipykernel=")) {
+        InterpreterContext.get().out.write("ipkernel is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("ipython=")) {
+        InterpreterContext.get().out.write("ipython is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("grpcio=")) {
+        InterpreterContext.get().out.write("grpcio is not installed\n".getBytes());
+        return false;
+      }
+      LOGGER.info("IPython prerequisite is meet");
+      return true;
+    } catch (Exception e) {
+      LOGGER.warn("Fail to checkIPythonPrerequisite", e);
+      return false;
+    }
+  }
+
+  private void setupJVMGateway(int jvmGatewayPort) throws IOException {
+    gatewayServer = new GatewayServer(this, jvmGatewayPort);
+    gatewayServer.start();
+
+    InputStream input =
+        getClass().getClassLoader().getResourceAsStream("grpc/python/zeppelin_python.py");
+    List<String> lines = IOUtils.readLines(input);
+    ExecuteResponse response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+        .setCode(StringUtils.join(lines, System.lineSeparator())
+            .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")).build());
+    if (response.getStatus() == ExecuteStatus.ERROR) {
+      throw new IOException("Fail to setup JVMGateway\n" + response.getOutput());
+    }
+
+    if (additionalPythonInitFile != null) {
+      input = getClass().getClassLoader().getResourceAsStream(additionalPythonInitFile);
+      lines = IOUtils.readLines(input);
+      response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+          .setCode(StringUtils.join(lines, System.lineSeparator())
+              .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")).build());
+      if (response.getStatus() == ExecuteStatus.ERROR) {
+        throw new IOException("Fail to run additional Python init file: "
+            + additionalPythonInitFile + "\n" + response.getOutput());
+      }
+    }
+  }
+
+
+  private void launchIPythonKernel(int ipythonPort)
+      throws IOException, URISyntaxException {
+    // copy the python scripts to a temp directory, then launch ipython kernel in that folder
+    File tmpPythonScriptFolder = Files.createTempDirectory("zeppelin_ipython").toFile();
+    String[] ipythonScripts = {"ipython_server.py", "ipython_pb2.py", "ipython_pb2_grpc.py"};
+    for (String ipythonScript : ipythonScripts) {
+      URL url = getClass().getClassLoader().getResource("grpc/python"
+          + "/" + ipythonScript);
+      FileUtils.copyURLToFile(url, new File(tmpPythonScriptFolder, ipythonScript));
+    }
+
+    CommandLine cmd = CommandLine.parse(pythonExecutable);
+    cmd.addArgument(tmpPythonScriptFolder.getAbsolutePath() + "/ipython_server.py");
+    cmd.addArgument(ipythonPort + "");
+    DefaultExecutor executor = new DefaultExecutor();
+    ProcessLogOutputStream processOutput = new ProcessLogOutputStream(LOGGER);
+    executor.setStreamHandler(new PumpStreamHandler(processOutput));
+    watchDog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
+    executor.setWatchdog(watchDog);
+
+    String py4jLibPath = null;
+    if (System.getenv("ZEPPELIN_HOME") != null) {
+      py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator
+          + PythonInterpreter.ZEPPELIN_PY4JPATH;
+    } else {
+      Path workingPath = Paths.get("..").toAbsolutePath();
+      py4jLibPath = workingPath + File.separator + PythonInterpreter.ZEPPELIN_PY4JPATH;
+    }
+    if (additionalPythonPath != null) {
+      // put the py4j at the end, because additionalPythonPath may already contain py4j.
+      // e.g. PySparkInterpreter
+      additionalPythonPath = additionalPythonPath + ":" + py4jLibPath;
+    } else {
+      additionalPythonPath = py4jLibPath;
+    }
+    Map<String, String> envs = EnvironmentUtils.getProcEnvironment();
+    if (envs.containsKey("PYTHONPATH")) {
+      envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH"));
+    } else {
+      envs.put("PYTHONPATH", additionalPythonPath);
+    }
+
+    LOGGER.debug("PYTHONPATH: " + envs.get("PYTHONPATH"));
+    executor.execute(cmd, envs, this);
+
+    // wait until IPython kernel is started or timeout
+    long startTime = System.currentTimeMillis();
+    while (true) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        LOGGER.error("Interrupted by something", e);
+      }
+
+      try {
+        StatusResponse response = ipythonClient.status(StatusRequest.newBuilder().build());
+        if (response.getStatus() == IPythonStatus.RUNNING) {
+          LOGGER.info("IPython Kernel is Running");
+          break;
+        } else {
+          LOGGER.info("Wait for IPython Kernel to be started");
+        }
+      } catch (Exception e) {
+        // ignore the exception, because is may happen when grpc server has not started yet.
+        LOGGER.info("Wait for IPython Kernel to be started");
+      }
+
+      if ((System.currentTimeMillis() - startTime) > ipythonLaunchTimeout) {
+        throw new IOException("Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000
+            + " seconds");
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    if (watchDog != null) {
+      LOGGER.debug("Kill IPython Process");
+      ipythonClient.stop(StopRequest.newBuilder().build());
+      watchDog.destroyProcess();
+      gatewayServer.shutdown();
+    }
+  }
+
+  @Override
+  public InterpreterResult interpret(String st, InterpreterContext context) {
+    zeppelinContext.setGui(context.getGui());
+    interpreterOutput.setInterpreterOutput(context.out);
+    ExecuteResponse response =
+        ipythonClient.stream_execute(ExecuteRequest.newBuilder().setCode(st).build(),
+            interpreterOutput);
+    try {
+      interpreterOutput.getInterpreterOutput().flush();
+    } catch (IOException e) {
+      throw new RuntimeException("Fail to write output", e);
+    }
+    InterpreterResult result = new InterpreterResult(
+        InterpreterResult.Code.valueOf(response.getStatus().name()));
+    return result;
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+    ipythonClient.cancel(CancelRequest.newBuilder().build());
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.SIMPLE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return 0;
+  }
+
+  @Override
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+                                                InterpreterContext interpreterContext) {
+    List<InterpreterCompletion> completions = new ArrayList<>();
+    CompletionResponse response =
+        ipythonClient.complete(
+            CompletionRequest.getDefaultInstance().newBuilder().setCode(buf)
+                .setCursor(cursor).build());
+    for (int i = 0; i < response.getMatchesCount(); i++) {
+      completions.add(new InterpreterCompletion(
+          response.getMatches(i), response.getMatches(i), ""));
+    }
+    return completions;
+  }
+
+  public PythonZeppelinContext getZeppelinContext() {
+    return zeppelinContext;
+  }
+
+  @Override
+  public void onProcessComplete(int exitValue) {
+    LOGGER.warn("Python Process is completed with exitValue: " + exitValue);
+  }
+
+  @Override
+  public void onProcessFailed(ExecuteException e) {
+    LOGGER.warn("Exception happens in Python Process", e);
+  }
+
+  private static class ProcessLogOutputStream extends LogOutputStream {
+
+    private Logger logger;
+
+    public ProcessLogOutputStream(Logger logger) {
+      this.logger = logger;
+    }
+
+    @Override
+    protected void processLine(String s, int i) {
+      this.logger.debug("Process Output: " + s);
+    }
+  }
+}

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -91,6 +91,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   private static final int MAX_TIMEOUT_SEC = 10;
 
   private long pythonPid = 0;
+  private IPythonInterpreter iPythonInterpreter;
 
   Integer statementSetNotifier = new Integer(0);
 
@@ -219,6 +220,37 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public void open() {
+    // try IPythonInterpreter first. If it is not available, we will fallback to the original
+    // python interpreter implementation.
+    iPythonInterpreter = getIPythonInterpreter();
+    if (getProperty().getProperty("zeppelin.python.useIPython", "true").equals("true") &&
+      iPythonInterpreter.checkIPythonPrerequisite()) {
+      try {
+        iPythonInterpreter.open();
+        if (InterpreterContext.get() != null) {
+          InterpreterContext.get().out.write(("IPython is available, " +
+              "use IPython for PythonInterpreter\n")
+              .getBytes());
+        }
+        LOG.info("Use IPythonInterpreter to replace PythonInterpreter");
+        return;
+      } catch (Exception e) {
+        iPythonInterpreter = null;
+      }
+    }
+    // reset iPythonInterpreter to null
+    iPythonInterpreter = null;
+
+    try {
+      if (InterpreterContext.get() != null) {
+        InterpreterContext.get().out.write(("IPython is not available, " +
+            "use the native PythonInterpreter\n")
+            .getBytes());
+      }
+    } catch (IOException e) {
+      LOG.warn("Fail to write InterpreterOutput", e.getMessage());
+    }
+
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
@@ -232,8 +264,27 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     }
   }
 
+  private IPythonInterpreter getIPythonInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    IPythonInterpreter ipython = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(IPythonInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    ipython = (IPythonInterpreter) p;
+    return ipython;
+  }
+
   @Override
   public void close() {
+    if (iPythonInterpreter != null) {
+      iPythonInterpreter.close();
+      return;
+    }
     pythonscriptRunning = false;
     pythonScriptInitialized = false;
 
@@ -319,6 +370,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.interpret(cmd, contextInterpreter);
+    }
     if (cmd == null || cmd.isEmpty()) {
       return new InterpreterResult(Code.SUCCESS, "");
     }
@@ -411,6 +465,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public void cancel(InterpreterContext context) {
+    if (iPythonInterpreter != null) {
+      iPythonInterpreter.cancel(context);
+    }
     try {
       interrupt();
     } catch (IOException e) {
@@ -425,11 +482,17 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public int getProgress(InterpreterContext context) {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.getProgress(context);
+    }
     return 0;
   }
 
   @Override
   public Scheduler getScheduler() {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.getScheduler();
+    }
     return SchedulerFactory.singleton().createOrGetFIFOScheduler(
         PythonInterpreter.class.getName() + this.hashCode());
   }
@@ -437,6 +500,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   @Override
   public List<InterpreterCompletion> completion(String buf, int cursor,
       InterpreterContext interpreterContext) {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.completion(buf, cursor, interpreterContext);
+    }
     return null;
   }
 

python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java

@@ -0,0 +1,49 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ZeppelinContext for Python
+ */
+public class PythonZeppelinContext extends BaseZeppelinContext {
+
+  public PythonZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+    super(hooks, maxResult);
+  }
+
+  @Override
+  public Map<String, String> getInterpreterClassMap() {
+    return null;
+  }
+
+  @Override
+  public List<Class> getSupportedClasses() {
+    return null;
+  }
+
+  @Override
+  protected String showData(Object obj) {
+    return null;
+  }
+}

python/src/main/proto/ipython.proto

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.apache.zeppelin.python.proto";
+option java_outer_classname = "IPythonProto";
+option objc_class_prefix = "IPython";
+
+package ipython;
+
+// The IPython service definition.
+service IPython {
+  // Sends code
+  rpc execute (ExecuteRequest) returns (stream ExecuteResponse) {}
+
+  // Get completion
+  rpc complete (CompletionRequest) returns (CompletionResponse) {}
+
+  // Cancel the running statement
+  rpc cancel (CancelRequest) returns (CancelResponse) {}
+
+  // Get ipython kernel status
+  rpc status (StatusRequest) returns (StatusResponse) {}
+
+  rpc stop(StopRequest) returns (StopResponse) {}
+}
+
+enum ExecuteStatus {
+  SUCCESS = 0;
+  ERROR = 1;
+}
+
+enum IPythonStatus {
+  STARTING = 0;
+  RUNNING = 1;
+}
+
+enum OutputType {
+  TEXT = 0;
+  IMAGE = 1;
+}
+
+// The request message containing the code
+message ExecuteRequest {
+  string code = 1;
+}
+
+// The response message containing the execution result.
+message ExecuteResponse {
+  ExecuteStatus status = 1;
+  OutputType type = 2;
+  string output = 3;
+}
+
+message CancelRequest {
+
+}
+
+message CancelResponse {
+
+}
+
+message CompletionRequest {
+  string code = 1;
+  int32 cursor = 2;
+}
+
+message CompletionResponse {
+  repeated string matches = 1;
+}
+
+message StatusRequest {
+
+}
+
+message StatusResponse {
+  IPythonStatus status = 1;
+}
+
+message StopRequest {
+
+}
+
+message StopResponse {
+
+}

python/src/main/resources/grpc/generate_rpc.sh

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+python -m grpc_tools.protoc -I../../proto --python_out=python --grpc_python_out=python ../../proto/ipython.proto

python/src/main/resources/grpc/python/ipython_client.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import grpc
+
+import ipython_pb2
+import ipython_pb2_grpc
+
+
+def run():
+    channel = grpc.insecure_channel('localhost:50053')
+    stub = ipython_pb2_grpc.IPythonStub(channel)
+    response = stub.execute(ipython_pb2.ExecuteRequest(code="import time\nfor i in range(1,4):\n\ttime.sleep(1)\n\tprint(i)\n" +
+                                                            "%matplotlib inline\nimport matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)"))
+    for r in response:
+        print("output:" + r.output)
+
+    response = stub.execute(ipython_pb2.ExecuteRequest(code="range?"))
+    for r in response:
+        print(r)
+
+if __name__ == '__main__':
+    run()

python/src/main/resources/grpc/python/ipython_pb2.py

@@ -0,0 +1,751 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: ipython.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='ipython.proto',
+  package='ipython',
+  syntax='proto3',
+  serialized_pb=_b('\n\ripython.proto\x12\x07ipython\"\x1e\n\x0e\x45xecuteRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\"l\n\x0f\x45xecuteResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.ExecuteStatus\x12!\n\x04type\x18\x02 \x01(\x0e\x32\x13.ipython.OutputType\x12\x0e\n\x06output\x18\x03 \x01(\t\"\x0f\n\rCancelRequest\"\x10\n\x0e\x43\x61ncelResponse\"1\n\x11\x43ompletionRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0e\n\x06\x63ursor\x18\x02 \x01(\x05\"%\n\x12\x43ompletionResponse\x12\x0f\n\x07matches\x18\x01 \x03(\t\"\x0f\n\rStatusRequest\"8\n\x0eStatusResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.IPythonStatus\"\r\n\x0bStopRequest\"\x0e\n\x0cStopResponse*\'\n\rExecuteStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\t\n\x05\x45RROR\x10\x01**\n\rIPythonStatus\x12\x0c\n\x08STARTING\x10\x00\x12\x0b\n\x07RUNNING\x10\x01*!\n\nOutputType\x12\x08\n\x04TEXT\x10\x00\x12\t\n\x05IMAGE\x10\x01\x32\xc3\x02\n\x07IPython\x12@\n\x07\x65xecute\x12\x17.ipython.ExecuteRequest\x1a\x18.ipython.ExecuteResponse\"\x00\x30\x01\x12\x45\n\x08\x63omplete\x12\x1a.ipython.CompletionRequest\x1a\x1b.ipython.CompletionResponse\"\x00\x12;\n\x06\x63\x61ncel\x12\x16.ipython.CancelRequest\x1a\x17.ipython.CancelResponse\"\x00\x12;\n\x06status\x12\x16.ipython.StatusRequest\x1a\x17.ipython.StatusResponse\"\x00\x12\x35\n\x04stop\x12\x14.ipython.StopRequest\x1a\x15.ipython.StopResponse\"\x00\x42<\n org.apache.zeppelin.python.protoB\x0cIPythonProtoP\x01\xa2\x02\x07IPythonb\x06proto3')
+)
+
+_EXECUTESTATUS = _descriptor.EnumDescriptor(
+  name='ExecuteStatus',
+  full_name='ipython.ExecuteStatus',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='SUCCESS', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='ERROR', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=399,
+  serialized_end=438,
+)
+_sym_db.RegisterEnumDescriptor(_EXECUTESTATUS)
+
+ExecuteStatus = enum_type_wrapper.EnumTypeWrapper(_EXECUTESTATUS)
+_IPYTHONSTATUS = _descriptor.EnumDescriptor(
+  name='IPythonStatus',
+  full_name='ipython.IPythonStatus',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='STARTING', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='RUNNING', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=440,
+  serialized_end=482,
+)
+_sym_db.RegisterEnumDescriptor(_IPYTHONSTATUS)
+
+IPythonStatus = enum_type_wrapper.EnumTypeWrapper(_IPYTHONSTATUS)
+_OUTPUTTYPE = _descriptor.EnumDescriptor(
+  name='OutputType',
+  full_name='ipython.OutputType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='TEXT', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='IMAGE', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=484,
+  serialized_end=517,
+)
+_sym_db.RegisterEnumDescriptor(_OUTPUTTYPE)
+
+OutputType = enum_type_wrapper.EnumTypeWrapper(_OUTPUTTYPE)
+SUCCESS = 0
+ERROR = 1
+STARTING = 0
+RUNNING = 1
+TEXT = 0
+IMAGE = 1
+
+
+
+_EXECUTEREQUEST = _descriptor.Descriptor(
+  name='ExecuteRequest',
+  full_name='ipython.ExecuteRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='ipython.ExecuteRequest.code', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=26,
+  serialized_end=56,
+)
+
+
+_EXECUTERESPONSE = _descriptor.Descriptor(
+  name='ExecuteResponse',
+  full_name='ipython.ExecuteResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='status', full_name='ipython.ExecuteResponse.status', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='type', full_name='ipython.ExecuteResponse.type', index=1,
+      number=2, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='output', full_name='ipython.ExecuteResponse.output', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=58,
+  serialized_end=166,
+)
+
+
+_CANCELREQUEST = _descriptor.Descriptor(
+  name='CancelRequest',
+  full_name='ipython.CancelRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=168,
+  serialized_end=183,
+)
+
+
+_CANCELRESPONSE = _descriptor.Descriptor(
+  name='CancelResponse',
+  full_name='ipython.CancelResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=185,
+  serialized_end=201,
+)
+
+
+_COMPLETIONREQUEST = _descriptor.Descriptor(
+  name='CompletionRequest',
+  full_name='ipython.CompletionRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='ipython.CompletionRequest.code', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='cursor', full_name='ipython.CompletionRequest.cursor', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=203,
+  serialized_end=252,
+)
+
+
+_COMPLETIONRESPONSE = _descriptor.Descriptor(
+  name='CompletionResponse',
+  full_name='ipython.CompletionResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='matches', full_name='ipython.CompletionResponse.matches', index=0,
+      number=1, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=254,
+  serialized_end=291,
+)
+
+
+_STATUSREQUEST = _descriptor.Descriptor(
+  name='StatusRequest',
+  full_name='ipython.StatusRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=293,
+  serialized_end=308,
+)
+
+
+_STATUSRESPONSE = _descriptor.Descriptor(
+  name='StatusResponse',
+  full_name='ipython.StatusResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='status', full_name='ipython.StatusResponse.status', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=310,
+  serialized_end=366,
+)
+
+
+_STOPREQUEST = _descriptor.Descriptor(
+  name='StopRequest',
+  full_name='ipython.StopRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=368,
+  serialized_end=381,
+)
+
+
+_STOPRESPONSE = _descriptor.Descriptor(
+  name='StopResponse',
+  full_name='ipython.StopResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=383,
+  serialized_end=397,
+)
+
+_EXECUTERESPONSE.fields_by_name['status'].enum_type = _EXECUTESTATUS
+_EXECUTERESPONSE.fields_by_name['type'].enum_type = _OUTPUTTYPE
+_STATUSRESPONSE.fields_by_name['status'].enum_type = _IPYTHONSTATUS
+DESCRIPTOR.message_types_by_name['ExecuteRequest'] = _EXECUTEREQUEST
+DESCRIPTOR.message_types_by_name['ExecuteResponse'] = _EXECUTERESPONSE
+DESCRIPTOR.message_types_by_name['CancelRequest'] = _CANCELREQUEST
+DESCRIPTOR.message_types_by_name['CancelResponse'] = _CANCELRESPONSE
+DESCRIPTOR.message_types_by_name['CompletionRequest'] = _COMPLETIONREQUEST
+DESCRIPTOR.message_types_by_name['CompletionResponse'] = _COMPLETIONRESPONSE
+DESCRIPTOR.message_types_by_name['StatusRequest'] = _STATUSREQUEST
+DESCRIPTOR.message_types_by_name['StatusResponse'] = _STATUSRESPONSE
+DESCRIPTOR.message_types_by_name['StopRequest'] = _STOPREQUEST
+DESCRIPTOR.message_types_by_name['StopResponse'] = _STOPRESPONSE
+DESCRIPTOR.enum_types_by_name['ExecuteStatus'] = _EXECUTESTATUS
+DESCRIPTOR.enum_types_by_name['IPythonStatus'] = _IPYTHONSTATUS
+DESCRIPTOR.enum_types_by_name['OutputType'] = _OUTPUTTYPE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ExecuteRequest = _reflection.GeneratedProtocolMessageType('ExecuteRequest', (_message.Message,), dict(
+  DESCRIPTOR = _EXECUTEREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.ExecuteRequest)
+  ))
+_sym_db.RegisterMessage(ExecuteRequest)
+
+ExecuteResponse = _reflection.GeneratedProtocolMessageType('ExecuteResponse', (_message.Message,), dict(
+  DESCRIPTOR = _EXECUTERESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.ExecuteResponse)
+  ))
+_sym_db.RegisterMessage(ExecuteResponse)
+
+CancelRequest = _reflection.GeneratedProtocolMessageType('CancelRequest', (_message.Message,), dict(
+  DESCRIPTOR = _CANCELREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CancelRequest)
+  ))
+_sym_db.RegisterMessage(CancelRequest)
+
+CancelResponse = _reflection.GeneratedProtocolMessageType('CancelResponse', (_message.Message,), dict(
+  DESCRIPTOR = _CANCELRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CancelResponse)
+  ))
+_sym_db.RegisterMessage(CancelResponse)
+
+CompletionRequest = _reflection.GeneratedProtocolMessageType('CompletionRequest', (_message.Message,), dict(
+  DESCRIPTOR = _COMPLETIONREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CompletionRequest)
+  ))
+_sym_db.RegisterMessage(CompletionRequest)
+
+CompletionResponse = _reflection.GeneratedProtocolMessageType('CompletionResponse', (_message.Message,), dict(
+  DESCRIPTOR = _COMPLETIONRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CompletionResponse)
+  ))
+_sym_db.RegisterMessage(CompletionResponse)
+
+StatusRequest = _reflection.GeneratedProtocolMessageType('StatusRequest', (_message.Message,), dict(
+  DESCRIPTOR = _STATUSREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StatusRequest)
+  ))
+_sym_db.RegisterMessage(StatusRequest)
+
+StatusResponse = _reflection.GeneratedProtocolMessageType('StatusResponse', (_message.Message,), dict(
+  DESCRIPTOR = _STATUSRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StatusResponse)
+  ))
+_sym_db.RegisterMessage(StatusResponse)
+
+StopRequest = _reflection.GeneratedProtocolMessageType('StopRequest', (_message.Message,), dict(
+  DESCRIPTOR = _STOPREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StopRequest)
+  ))
+_sym_db.RegisterMessage(StopRequest)
+
+StopResponse = _reflection.GeneratedProtocolMessageType('StopResponse', (_message.Message,), dict(
+  DESCRIPTOR = _STOPRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StopResponse)
+  ))
+_sym_db.RegisterMessage(StopResponse)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n org.apache.zeppelin.python.protoB\014IPythonProtoP\001\242\002\007IPython'))
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+
+
+  class IPythonStub(object):
+    """The IPython service definition.
+    """
+
+    def __init__(self, channel):
+      """Constructor.
+
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.execute = channel.unary_stream(
+          '/ipython.IPython/execute',
+          request_serializer=ExecuteRequest.SerializeToString,
+          response_deserializer=ExecuteResponse.FromString,
+          )
+      self.complete = channel.unary_unary(
+          '/ipython.IPython/complete',
+          request_serializer=CompletionRequest.SerializeToString,
+          response_deserializer=CompletionResponse.FromString,
+          )
+      self.cancel = channel.unary_unary(
+          '/ipython.IPython/cancel',
+          request_serializer=CancelRequest.SerializeToString,
+          response_deserializer=CancelResponse.FromString,
+          )
+      self.status = channel.unary_unary(
+          '/ipython.IPython/status',
+          request_serializer=StatusRequest.SerializeToString,
+          response_deserializer=StatusResponse.FromString,
+          )
+      self.stop = channel.unary_unary(
+          '/ipython.IPython/stop',
+          request_serializer=StopRequest.SerializeToString,
+          response_deserializer=StopResponse.FromString,
+          )
+
+
+  class IPythonServicer(object):
+    """The IPython service definition.
+    """
+
+    def execute(self, request, context):
+      """Sends code
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def complete(self, request, context):
+      """Get completion
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def cancel(self, request, context):
+      """Cancel the running statement
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def status(self, request, context):
+      """Get ipython kernel status
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def stop(self, request, context):
+      # missing associated documentation comment in .proto file
+      pass
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_IPythonServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'execute': grpc.unary_stream_rpc_method_handler(
+            servicer.execute,
+            request_deserializer=ExecuteRequest.FromString,
+            response_serializer=ExecuteResponse.SerializeToString,
+        ),
+        'complete': grpc.unary_unary_rpc_method_handler(
+            servicer.complete,
+            request_deserializer=CompletionRequest.FromString,
+            response_serializer=CompletionResponse.SerializeToString,
+        ),
+        'cancel': grpc.unary_unary_rpc_method_handler(
+            servicer.cancel,
+            request_deserializer=CancelRequest.FromString,
+            response_serializer=CancelResponse.SerializeToString,
+        ),
+        'status': grpc.unary_unary_rpc_method_handler(
+            servicer.status,
+            request_deserializer=StatusRequest.FromString,
+            response_serializer=StatusResponse.SerializeToString,
+        ),
+        'stop': grpc.unary_unary_rpc_method_handler(
+            servicer.stop,
+            request_deserializer=StopRequest.FromString,
+            response_serializer=StopResponse.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'ipython.IPython', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaIPythonServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The IPython service definition.
+    """
+    def execute(self, request, context):
+      """Sends code
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def complete(self, request, context):
+      """Get completion
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def cancel(self, request, context):
+      """Cancel the running statement
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def status(self, request, context):
+      """Get ipython kernel status
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def stop(self, request, context):
+      # missing associated documentation comment in .proto file
+      pass
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+  class BetaIPythonStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The IPython service definition.
+    """
+    def execute(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Sends code
+      """
+      raise NotImplementedError()
+    def complete(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Get completion
+      """
+      raise NotImplementedError()
+    complete.future = None
+    def cancel(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Cancel the running statement
+      """
+      raise NotImplementedError()
+    cancel.future = None
+    def status(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Get ipython kernel status
+      """
+      raise NotImplementedError()
+    status.future = None
+    def stop(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      # missing associated documentation comment in .proto file
+      pass
+      raise NotImplementedError()
+    stop.future = None
+
+
+  def beta_create_IPython_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('ipython.IPython', 'cancel'): CancelRequest.FromString,
+      ('ipython.IPython', 'complete'): CompletionRequest.FromString,
+      ('ipython.IPython', 'execute'): ExecuteRequest.FromString,
+      ('ipython.IPython', 'status'): StatusRequest.FromString,
+      ('ipython.IPython', 'stop'): StopRequest.FromString,
+    }
+    response_serializers = {
+      ('ipython.IPython', 'cancel'): CancelResponse.SerializeToString,
+      ('ipython.IPython', 'complete'): CompletionResponse.SerializeToString,
+      ('ipython.IPython', 'execute'): ExecuteResponse.SerializeToString,
+      ('ipython.IPython', 'status'): StatusResponse.SerializeToString,
+      ('ipython.IPython', 'stop'): StopResponse.SerializeToString,
+    }
+    method_implementations = {
+      ('ipython.IPython', 'cancel'): face_utilities.unary_unary_inline(servicer.cancel),
+      ('ipython.IPython', 'complete'): face_utilities.unary_unary_inline(servicer.complete),
+      ('ipython.IPython', 'execute'): face_utilities.unary_stream_inline(servicer.execute),
+      ('ipython.IPython', 'status'): face_utilities.unary_unary_inline(servicer.status),
+      ('ipython.IPython', 'stop'): face_utilities.unary_unary_inline(servicer.stop),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_IPython_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('ipython.IPython', 'cancel'): CancelRequest.SerializeToString,
+      ('ipython.IPython', 'complete'): CompletionRequest.SerializeToString,
+      ('ipython.IPython', 'execute'): ExecuteRequest.SerializeToString,
+      ('ipython.IPython', 'status'): StatusRequest.SerializeToString,
+      ('ipython.IPython', 'stop'): StopRequest.SerializeToString,
+    }
+    response_deserializers = {
+      ('ipython.IPython', 'cancel'): CancelResponse.FromString,
+      ('ipython.IPython', 'complete'): CompletionResponse.FromString,
+      ('ipython.IPython', 'execute'): ExecuteResponse.FromString,
+      ('ipython.IPython', 'status'): StatusResponse.FromString,
+      ('ipython.IPython', 'stop'): StopResponse.FromString,
+    }
+    cardinalities = {
+      'cancel': cardinality.Cardinality.UNARY_UNARY,
+      'complete': cardinality.Cardinality.UNARY_UNARY,
+      'execute': cardinality.Cardinality.UNARY_STREAM,
+      'status': cardinality.Cardinality.UNARY_UNARY,
+      'stop': cardinality.Cardinality.UNARY_UNARY,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'ipython.IPython', cardinalities, options=stub_options)
+except ImportError:
+  pass
+# @@protoc_insertion_point(module_scope)

python/src/main/resources/grpc/python/ipython_pb2_grpc.py

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+import ipython_pb2 as ipython__pb2
+
+
+class IPythonStub(object):
+  """The IPython service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.execute = channel.unary_stream(
+        '/ipython.IPython/execute',
+        request_serializer=ipython__pb2.ExecuteRequest.SerializeToString,
+        response_deserializer=ipython__pb2.ExecuteResponse.FromString,
+        )
+    self.complete = channel.unary_unary(
+        '/ipython.IPython/complete',
+        request_serializer=ipython__pb2.CompletionRequest.SerializeToString,
+        response_deserializer=ipython__pb2.CompletionResponse.FromString,
+        )
+    self.cancel = channel.unary_unary(
+        '/ipython.IPython/cancel',
+        request_serializer=ipython__pb2.CancelRequest.SerializeToString,
+        response_deserializer=ipython__pb2.CancelResponse.FromString,
+        )
+    self.status = channel.unary_unary(
+        '/ipython.IPython/status',
+        request_serializer=ipython__pb2.StatusRequest.SerializeToString,
+        response_deserializer=ipython__pb2.StatusResponse.FromString,
+        )
+    self.stop = channel.unary_unary(
+        '/ipython.IPython/stop',
+        request_serializer=ipython__pb2.StopRequest.SerializeToString,
+        response_deserializer=ipython__pb2.StopResponse.FromString,
+        )
+
+
+class IPythonServicer(object):
+  """The IPython service definition.
+  """
+
+  def execute(self, request, context):
+    """Sends code
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def complete(self, request, context):
+    """Get completion
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def cancel(self, request, context):
+    """Cancel the running statement
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def status(self, request, context):
+    """Get ipython kernel status
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def stop(self, request, context):
+    # missing associated documentation comment in .proto file
+    pass
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_IPythonServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'execute': grpc.unary_stream_rpc_method_handler(
+          servicer.execute,
+          request_deserializer=ipython__pb2.ExecuteRequest.FromString,
+          response_serializer=ipython__pb2.ExecuteResponse.SerializeToString,
+      ),
+      'complete': grpc.unary_unary_rpc_method_handler(
+          servicer.complete,
+          request_deserializer=ipython__pb2.CompletionRequest.FromString,
+          response_serializer=ipython__pb2.CompletionResponse.SerializeToString,
+      ),
+      'cancel': grpc.unary_unary_rpc_method_handler(
+          servicer.cancel,
+          request_deserializer=ipython__pb2.CancelRequest.FromString,
+          response_serializer=ipython__pb2.CancelResponse.SerializeToString,
+      ),
+      'status': grpc.unary_unary_rpc_method_handler(
+          servicer.status,
+          request_deserializer=ipython__pb2.StatusRequest.FromString,
+          response_serializer=ipython__pb2.StatusResponse.SerializeToString,
+      ),
+      'stop': grpc.unary_unary_rpc_method_handler(
+          servicer.stop,
+          request_deserializer=ipython__pb2.StopRequest.FromString,
+          response_serializer=ipython__pb2.StopResponse.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'ipython.IPython', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

python/src/main/resources/grpc/python/ipython_server.py

@@ -0,0 +1,155 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import jupyter_client
+import sys
+import threading
+import time
+from concurrent import futures
+
+import grpc
+import ipython_pb2
+import ipython_pb2_grpc
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+is_py2 = sys.version[0] == '2'
+if is_py2:
+    import Queue as queue
+else:
+    import queue as queue
+
+
+TIMEOUT = 30
+
+class IPython(ipython_pb2_grpc.IPythonServicer):
+
+    def __init__(self, server):
+        self._status = ipython_pb2.STARTING
+        self._server = server
+
+    def start(self):
+        print("starting...")
+        sys.stdout.flush()
+        self._km, self._kc = jupyter_client.manager.start_new_kernel(kernel_name='python')
+        self._status = ipython_pb2.RUNNING
+
+    def execute(self, request, context):
+        print("execute code: " + request.code)
+        sys.stdout.flush()
+        stdout_queue = queue.Queue(maxsize = 10)
+        stderr_queue = queue.Queue(maxsize = 10)
+        image_queue = queue.Queue(maxsize = 5)
+
+        def _output_hook(msg):
+            msg_type = msg['header']['msg_type']
+            content = msg['content']
+            if msg_type == 'stream':
+                stdout_queue.put(content['text'])
+            elif msg_type in ('display_data', 'execute_result'):
+                stdout_queue.put(content['data'].get('text/plain', ''))
+                if 'image/png' in content['data']:
+                    image_queue.put(content['data']['image/png'])
+            elif msg_type == 'error':
+                stderr_queue.put('\n'.join(content['traceback']))
+
+
+        payload_reply = []
+        def execute_worker():
+            reply = self._kc.execute_interactive(request.code,
+                                            output_hook=_output_hook,
+                                            timeout=TIMEOUT)
+            payload_reply.append(reply)
+
+        t = threading.Thread(name="ConsumerThread", target=execute_worker)
+        t.start()
+
+        while t.is_alive():
+            while not stdout_queue.empty():
+                output = stdout_queue.get()
+                yield ipython_pb2.ExecuteResponse(status=ipython_pb2.SUCCESS,
+                                                  type=ipython_pb2.TEXT,
+                                                  output=output)
+            while not stderr_queue.empty():
+                output = stderr_queue.get()
+                yield ipython_pb2.ExecuteResponse(status=ipython_pb2.ERROR,
+                                                  type=ipython_pb2.TEXT,
+                                                  output=output)
+            while not image_queue.empty():
+                output = image_queue.get()
+                yield ipython_pb2.ExecuteResponse(status=ipython_pb2.SUCCESS,
+                                                  type=ipython_pb2.IMAGE,
+                                                  output=output)
+
+        while not stdout_queue.empty():
+            output = stdout_queue.get()
+            yield ipython_pb2.ExecuteResponse(status=ipython_pb2.SUCCESS,
+                                              type=ipython_pb2.TEXT,
+                                              output=output)
+        while not stderr_queue.empty():
+            output = stderr_queue.get()
+            yield ipython_pb2.ExecuteResponse(status=ipython_pb2.ERROR,
+                                              type=ipython_pb2.TEXT,
+                                              output=output)
+        while not image_queue.empty():
+            output = image_queue.get()
+            yield ipython_pb2.ExecuteResponse(status=ipython_pb2.SUCCESS,
+                                              type=ipython_pb2.IMAGE,
+                                              output=output)
+
+        if payload_reply:
+            result = []
+            for payload in payload_reply[0]['content']['payload']:
+                if payload['data']['text/plain']:
+                    result.append(payload['data']['text/plain'])
+            if result:
+                yield ipython_pb2.ExecuteResponse(status=ipython_pb2.SUCCESS,
+                                                  type=ipython_pb2.TEXT,
+                                                  output='\n'.join(result))
+
+    def cancel(self, request, context):
+        self._km.interrupt_kernel()
+        return ipython_pb2.CancelResponse()
+
+    def complete(self, request, context):
+        reply = self._kc.complete(request.code, request.cursor, reply=True, timeout=TIMEOUT)
+        return ipython_pb2.CompletionResponse(matches=reply['content']['matches'])
+
+    def status(self, request, context):
+        return ipython_pb2.StatusResponse(status = self._status)
+
+    def stop(self, request, context):
+        self._server.stop(0)
+        sys.exit(0)
+
+
+def serve(port):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    ipython = IPython(server)
+    ipython_pb2_grpc.add_IPythonServicer_to_server(ipython, server)
+    server.add_insecure_port('[::]:' + port)
+    server.start()
+    ipython.start()
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+if __name__ == '__main__':
+    serve(sys.argv[1])

python/src/main/resources/grpc/python/zeppelin_python.py

@@ -0,0 +1,107 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from py4j.java_gateway import java_import, JavaGateway, GatewayClient
+
+from io import BytesIO
+try:
+  from StringIO import StringIO
+except ImportError:
+  from io import StringIO
+
+class PyZeppelinContext(object):
+  """ A context impl that uses Py4j to communicate to JVM
+  """
+
+  def __init__(self, z):
+    self.z = z
+    self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
+    self.javaList = gateway.jvm.java.util.ArrayList
+    self.max_result = z.getMaxResult()
+
+  def input(self, name, defaultValue=""):
+    return self.z.getGui().input(name, defaultValue)
+
+  def select(self, name, options, defaultValue=""):
+    javaOptions = gateway.new_array(self.paramOption, len(options))
+    i = 0
+    for tuple in options:
+      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
+      i += 1
+    return self.z.getGui().select(name, defaultValue, javaOptions)
+
+  def checkbox(self, name, options, defaultChecked=[]):
+    javaOptions = gateway.new_array(self.paramOption, len(options))
+    i = 0
+    for tuple in options:
+      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
+      i += 1
+    javaDefaultCheck = self.javaList()
+    for check in defaultChecked:
+      javaDefaultCheck.append(check)
+    return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
+
+  def show(self, p, **kwargs):
+    if type(p).__name__ == "DataFrame": # does not play well with sub-classes
+      # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
+      # and so a dependency on pandas
+      self.show_dataframe(p, **kwargs)
+    elif hasattr(p, '__call__'):
+      p() #error reporting
+
+  def show_dataframe(self, df, show_index=False, **kwargs):
+    """Pretty prints DF using Table Display System
+    """
+    limit = len(df) > self.max_result
+    header_buf = StringIO("")
+    if show_index:
+      idx_name = str(df.index.name) if df.index.name is not None else ""
+      header_buf.write(idx_name + "\t")
+    header_buf.write(str(df.columns[0]))
+    for col in df.columns[1:]:
+      header_buf.write("\t")
+      header_buf.write(str(col))
+    header_buf.write("\n")
+
+    body_buf = StringIO("")
+    rows = df.head(self.max_result).values if limit else df.values
+    index = df.index.values
+    for idx, row in zip(index, rows):
+      if show_index:
+        body_buf.write("%html <strong>{}</strong>".format(idx))
+        body_buf.write("\t")
+      body_buf.write(str(row[0]))
+      for cell in row[1:]:
+        body_buf.write("\t")
+        body_buf.write(str(cell))
+      body_buf.write("\n")
+    body_buf.seek(0); header_buf.seek(0)
+    #TODO(bzz): fix it, so it shows red notice, as in Spark
+    print("%table " + header_buf.read() + body_buf.read()) # +
+    #      ("\n<font color=red>Results are limited by {}.</font>" \
+    #          .format(self.max_result) if limit else "")
+    #)
+    body_buf.close(); header_buf.close()
+
+
+# start JVM gateway
+client = GatewayClient(address='127.0.0.1', port=${JVM_GATEWAY_PORT})
+gateway = JavaGateway(client)
+java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
+intp = gateway.entry_point
+z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
+

python/src/main/resources/interpreter-setting.json

@@ -17,6 +17,29 @@
         "defaultValue": "1000",
         "description": "Max number of dataframe rows to display.",
         "type": "number"
+      },
+      "zeppelin.python.useIPython": {
+        "propertyName": "zeppelin.python.useIPython",
+        "defaultValue": true,
+        "description": "whether use IPython when it is available",
+        "type": "checkbox"
+      }
+    },
+    "editor": {
+      "language": "python",
+      "editOnDblClick": false
+    }
+  },
+  {
+    "group": "python",
+    "name": "ipython",
+    "className": "org.apache.zeppelin.python.IPythonInterpreter",
+    "properties": {
+      "zeppelin.ipython.launch.timeout": {
+        "propertyName": "zeppelin.ipython.launch.timeout",
+        "defaultValue": "30000",
+        "description": "time out for ipython launch",
+        "type": "number"
       }
     },
     "editor": {

python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java

@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.python;
+
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.display.ui.CheckBox;
+import org.apache.zeppelin.display.ui.Select;
+import org.apache.zeppelin.display.ui.TextBox;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterOutputListener;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+
+
+public class IPythonInterpreterTest {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreterTest.class);
+  private IPythonInterpreter interpreter;
+
+  @Before
+  public void setUp() {
+    Properties properties = new Properties();
+    interpreter = new IPythonInterpreter(properties);
+    InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class);
+    interpreter.setInterpreterGroup(mockInterpreterGroup);
+    interpreter.open();
+  }
+
+  @After
+  public void close() {
+    interpreter.close();
+  }
+
+
+  @Test
+  public void testIPython() throws IOException, InterruptedException {
+    testInterpreter(interpreter);
+  }
+
+  public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException {
+    // to make this test can run under both python2 and python3
+    InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+
+    // single output without print
+    InterpreterContext context = getInterpreterContext();
+    result = interpreter.interpret("'hello world'", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    List<InterpreterResultMessage> interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("'hello world'", interpreterResultMessages.get(0).getData());
+
+    // only the last statement is printed
+    context = getInterpreterContext();
+    result = interpreter.interpret("'hello world'\n'hello world2'", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("'hello world2'", interpreterResultMessages.get(0).getData());
+
+    // single output
+    context = getInterpreterContext();
+    result = interpreter.interpret("print('hello world')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("hello world\n", interpreterResultMessages.get(0).getData());
+
+    // multiple output
+    context = getInterpreterContext();
+    result = interpreter.interpret("print('hello world')\nprint('hello world2')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("hello world\nhello world2\n", interpreterResultMessages.get(0).getData());
+
+    // assignment
+    context = getInterpreterContext();
+    result = interpreter.interpret("abc=1",context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(0, interpreterResultMessages.size());
+
+    // if block
+    context = getInterpreterContext();
+    result = interpreter.interpret("if abc > 0:\n\tprint('True')\nelse:\n\tprint('False')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("True\n", interpreterResultMessages.get(0).getData());
+
+    // for loop
+    context = getInterpreterContext();
+    result = interpreter.interpret("for i in range(3):\n\tprint(i)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("0\n1\n2\n", interpreterResultMessages.get(0).getData());
+
+    // syntax error
+    context = getInterpreterContext();
+    result = interpreter.interpret("print(unknown)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.ERROR, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertTrue(interpreterResultMessages.get(0).getData().contains("name 'unknown' is not defined"));
+
+    // raise runtime exception
+    context = getInterpreterContext();
+    result = interpreter.interpret("1/0", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.ERROR, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertTrue(interpreterResultMessages.get(0).getData().contains("ZeroDivisionError"));
+
+    // ZEPPELIN-1133
+    context = getInterpreterContext();
+    result = interpreter.interpret("def greet(name):\n" +
+        "    print('Hello', name)\n" +
+        "greet('Jack')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("Hello Jack\n",interpreterResultMessages.get(0).getData());
+
+    // ZEPPELIN-1114
+    context = getInterpreterContext();
+    result = interpreter.interpret("print('there is no Error: ok')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals("there is no Error: ok\n", interpreterResultMessages.get(0).getData());
+
+    // completion
+    context = getInterpreterContext();
+    List<InterpreterCompletion> completions = interpreter.completion("ab", 2, context);
+    assertEquals(2, completions.size());
+    assertEquals("abc", completions.get(0).getValue());
+    assertEquals("abs", completions.get(1).getValue());
+
+    context = getInterpreterContext();
+    interpreter.interpret("import sys", context);
+    completions = interpreter.completion("sys.", 4, context);
+    assertFalse(completions.isEmpty());
+
+    context = getInterpreterContext();
+    completions = interpreter.completion("sys.std", 7, context);
+    assertEquals(3, completions.size());
+    assertEquals("sys.stderr", completions.get(0).getValue());
+    assertEquals("sys.stdin", completions.get(1).getValue());
+    assertEquals("sys.stdout", completions.get(2).getValue());
+
+    // there's no completion for 'a.' because it is not recognized by compiler for now.
+    context = getInterpreterContext();
+    String st = "a='hello'\na.";
+    completions = interpreter.completion(st, st.length(), context);
+    assertEquals(0, completions.size());
+
+    // define `a` first
+    context = getInterpreterContext();
+    st = "a='hello'";
+    result = interpreter.interpret(st, context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(0, interpreterResultMessages.size());
+
+    // now we can get the completion for `a.`
+    context = getInterpreterContext();
+    st = "a.";
+    completions = interpreter.completion(st, st.length(), context);
+    // it is different for python2 and python3 and may even different for different minor version
+    // so only verify it is larger than 20
+    assertTrue(completions.size() > 20);
+
+    context = getInterpreterContext();
+    st = "a.co";
+    completions = interpreter.completion(st, st.length(), context);
+    assertEquals(1, completions.size());
+    assertEquals("a.count", completions.get(0).getValue());
+
+    // cursor is in the middle of code
+    context = getInterpreterContext();
+    st = "a.co\b='hello";
+    completions = interpreter.completion(st, 4, context);
+    assertEquals(1, completions.size());
+    assertEquals("a.count", completions.get(0).getValue());
+
+    // ipython help
+    context = getInterpreterContext();
+    result = interpreter.interpret("range?", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertTrue(interpreterResultMessages.get(0).getData().contains("range(stop)"));
+
+    // timeit
+    context = getInterpreterContext();
+    result = interpreter.interpret("%timeit range(100)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertTrue(interpreterResultMessages.get(0).getData().contains("loops"));
+
+    // cancel
+    final InterpreterContext context2 = getInterpreterContext();
+    new Thread() {
+      @Override
+      public void run() {
+        try {
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        interpreter.cancel(context2);
+      }
+    }.start();
+    result = interpreter.interpret("import time\ntime.sleep(10)", context2);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.ERROR, result.code());
+    interpreterResultMessages = context2.out.getInterpreterResultMessages();
+    assertTrue(interpreterResultMessages.get(0).getData().contains("KeyboardInterrupt"));
+
+    // matplotlib
+    context = getInterpreterContext();
+    result = interpreter.interpret("%matplotlib inline\nimport matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    // the order of IMAGE and TEXT is not determined
+    // check there must be one IMAGE output
+    boolean hasImageOutput = false;
+    for (InterpreterResultMessage msg : interpreterResultMessages) {
+      if (msg.getType() == InterpreterResult.Type.IMG) {
+        hasImageOutput = true;
+      }
+    }
+    assertTrue("No Image Output", hasImageOutput);
+
+    // bokeh
+    // bokeh initialization
+    context = getInterpreterContext();
+    result = interpreter.interpret("from bokeh.io import output_notebook, show\n" +
+        "from bokeh.plotting import figure\n" +
+        "output_notebook(notebook_type='zeppelin')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(2, interpreterResultMessages.size());
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
+    assertTrue(interpreterResultMessages.get(0).getData().contains("Loading BokehJS"));
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
+    assertTrue(interpreterResultMessages.get(1).getData().contains("BokehJS is being loaded"));
+
+    // bokeh plotting
+    context = getInterpreterContext();
+    result = interpreter.interpret("from bokeh.plotting import figure, output_file, show\n" +
+        "x = [1, 2, 3, 4, 5]\n" +
+        "y = [6, 7, 2, 4, 5]\n" +
+        "p = figure(title=\"simple line example\", x_axis_label='x', y_axis_label='y')\n" +
+        "p.line(x, y, legend=\"Temp.\", line_width=2)\n" +
+        "show(p)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
+    // docs_json is the source data of plotting which bokeh would use to render the plotting.
+    assertTrue(interpreterResultMessages.get(0).getData().contains("docs_json"));
+
+    // ggplot
+    context = getInterpreterContext();
+    result = interpreter.interpret("from ggplot import *\n" +
+        "ggplot(diamonds, aes(x='price', fill='cut')) +\\\n" +
+        "    geom_density(alpha=0.25) +\\\n" +
+        "    facet_wrap(\"clarity\")", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    // the order of IMAGE and TEXT is not determined
+    // check there must be one IMAGE output
+    hasImageOutput = false;
+    for (InterpreterResultMessage msg : interpreterResultMessages) {
+      if (msg.getType() == InterpreterResult.Type.IMG) {
+        hasImageOutput = true;
+      }
+    }
+    assertTrue("No Image Output", hasImageOutput);
+
+    // ZeppelinContext
+
+    // TextBox
+    context = getInterpreterContext();
+    result = interpreter.interpret("z.input(name='text_1', defaultValue='value_1')", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertTrue(interpreterResultMessages.get(0).getData().contains("'value_1'"));
+    assertEquals(1, context.getGui().getForms().size());
+    assertTrue(context.getGui().getForms().get("text_1") instanceof TextBox);
+    TextBox textbox = (TextBox) context.getGui().getForms().get("text_1");
+    assertEquals("text_1", textbox.getName());
+    assertEquals("value_1", textbox.getDefaultValue());
+
+    // Select
+    context = getInterpreterContext();
+    result = interpreter.interpret("z.select(name='select_1', options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    assertEquals(1, context.getGui().getForms().size());
+    assertTrue(context.getGui().getForms().get("select_1") instanceof Select);
+    Select select = (Select) context.getGui().getForms().get("select_1");
+    assertEquals("select_1", select.getName());
+    assertEquals(2, select.getOptions().length);
+    assertEquals("name_1", select.getOptions()[0].getDisplayName());
+    assertEquals("value_1", select.getOptions()[0].getValue());
+
+    // CheckBox
+    context = getInterpreterContext();
+    result = interpreter.interpret("z.checkbox(name='checkbox_1', options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    assertEquals(1, context.getGui().getForms().size());
+    assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox);
+    CheckBox checkbox = (CheckBox) context.getGui().getForms().get("checkbox_1");
+    assertEquals("checkbox_1", checkbox.getName());
+    assertEquals(2, checkbox.getOptions().length);
+    assertEquals("name_1", checkbox.getOptions()[0].getDisplayName());
+    assertEquals("value_1", checkbox.getOptions()[0].getValue());
+
+    // Pandas DataFrame
+    context = getInterpreterContext();
+    result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType());
+    assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData());
+  }
+
+  private static InterpreterContext getInterpreterContext() {
+    return new InterpreterContext(
+        "noteId",
+        "paragraphId",
+        "replName",
+        "paragraphTitle",
+        "paragraphText",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        null,
+        null,
+        null,
+        new InterpreterOutput(null));
+  }
+}

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java

@@ -53,6 +53,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
     Properties p = new Properties();
     p.setProperty("zeppelin.python", "python");
     p.setProperty("zeppelin.python.maxResult", "100");
+    p.setProperty("zeppelin.python.useIPython", "false");
 
     intpGroup = new InterpreterGroup();
 

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java

@@ -73,9 +73,21 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     Properties p = new Properties();
     p.setProperty("zeppelin.python", "python");
     p.setProperty("zeppelin.python.maxResult", "100");
+    p.setProperty("zeppelin.python.useIPython", "false");
 
     intpGroup = new InterpreterGroup();
 
+    out = new InterpreterOutput(this);
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        out);
+    InterpreterContext.set(context);
+
     python = new PythonInterpreter(p);
     python.setInterpreterGroup(intpGroup);
     python.open();
@@ -85,16 +97,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
 
     intpGroup.put("note", Arrays.asList(python, sql));
 
-    out = new InterpreterOutput(this);
 
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        out);
 
     // to make sure python is running.
     InterpreterResult ret = python.interpret("\n", context);
@@ -140,10 +143,10 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     ret = sql.interpret("select name, age from df2 where age < 40", context);
 
     //then
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
-    assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("moon\t33") > 0);
-    assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("park\t34") > 0);
+    assertEquals(new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType());
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("moon\t33") > 0);
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("park\t34") > 0);
 
     assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from df2", context).code());
   }
@@ -156,7 +159,6 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     //then
     assertNotNull("Interpreter returned 'null'", ret);
     assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(out.toInterpreterResultMessage().size() == 0);
   }
 
   @Test
@@ -176,10 +178,10 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
 
     // then
     assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
-    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("index_name"));
-    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("nan"));
-    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("6.7"));
+    assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType());
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("index_name"));
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("nan"));
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("6.7"));
   }
 
   @Override

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java

@@ -59,6 +59,7 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
     Properties p = new Properties();
     p.setProperty(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON);
     p.setProperty(MAX_RESULT, "1000");
+    p.setProperty("zeppelin.python.useIPython", "false");
     return p;
   }
 
@@ -85,6 +86,7 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
         new LocalResourcePool("id"),
         new LinkedList<InterpreterContextRunner>(),
         out);
+    InterpreterContext.set(context);
     pythonInterpreter.open();
   }
 

python/src/test/resources/log4j.properties

@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=
+#%5p [%t] (%F:%L) - %m%n
+#%-4r [%t] %-5p %c %x - %m%n
+#
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+log4j.logger.org.apache.zeppelin.python.IPythonInterpreter=DEBUG
+log4j.logger.org.apache.zeppelin.python.IPythonClient=DEBUG

scripts/docker/zeppelin/bin/Dockerfile

@@ -39,14 +39,11 @@ RUN echo "$LOG_TAG install tini related packages" && \
     dpkg -i tini.deb && \
     rm tini.deb
 
-ENV JAVA_HOME=/usr/lib/jvm/java-8-oracle
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
 RUN echo "$LOG_TAG Install java8" && \
-    echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
-    add-apt-repository -y ppa:webupd8team/java && \
     apt-get -y update && \
-    apt-get install -y oracle-java8-installer && \
-    rm -rf /var/lib/apt/lists/* && \
-    rm -rf /var/cache/oracle-jdk8-installer
+    apt-get install -y openjdk-8-jdk && \
+    rm -rf /var/lib/apt/lists/*
 
 # should install conda first before numpy, matploylib since pip and python will be installed by conda
 RUN echo "$LOG_TAG Install miniconda2 related packages" && \

shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java

@@ -20,6 +20,7 @@ package org.apache.zeppelin.shell;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.File;
 import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
@@ -47,6 +48,7 @@ import org.slf4j.LoggerFactory;
 public class ShellInterpreter extends KerberosInterpreter {
   private static final Logger LOGGER = LoggerFactory.getLogger(ShellInterpreter.class);
   private static final String TIMEOUT_PROPERTY = "shell.command.timeout.millisecs";
+  private static final String DIRECTORY_USER_HOME = "shell.working.directory.user.home";
   private final boolean isWindows = System.getProperty("os.name").startsWith("Windows");
   private final String shell = isWindows ? "cmd /c" : "bash -c";
   ConcurrentHashMap<String, DefaultExecutor> executors;
@@ -98,6 +100,10 @@ public class ShellInterpreter extends KerberosInterpreter {
         contextInterpreter.out, contextInterpreter.out));
       executor.setWatchdog(new ExecuteWatchdog(Long.valueOf(getProperty(TIMEOUT_PROPERTY))));
       executors.put(contextInterpreter.getParagraphId(), executor);
+      if (Boolean.valueOf(getProperty(DIRECTORY_USER_HOME))) {
+        executor.setWorkingDirectory(new File(System.getProperty("user.home")));
+      }
+
       int exitVal = executor.execute(cmdLine);
       LOGGER.info("Paragraph " + contextInterpreter.getParagraphId() 
         + " return with exit value: " + exitVal);

shell/src/main/resources/interpreter-setting.json

@@ -11,6 +11,13 @@
         "description": "Shell command time out in millisecs. Default = 60000",
         "type": "number"
       },
+      "shell.working.directory.user.home": {
+        "envName": "SHELL_WORKING_DIRECTORY_USER_HOME",
+        "propertyName": "shell.working.directory.user.home",
+        "defaultValue": false,
+        "description": "If this set to true, the shell's working directory will be set to user home",
+        "type": "checkbox"
+      },
       "zeppelin.shell.auth.type": {
         "envName": null,
         "propertyName": "zeppelin.shell.auth.type",

spark/pom.xml

@@ -71,6 +71,32 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zeppelin-python</artifactId>
+      <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>net.sf.py4j</groupId>
+          <artifactId>py4j</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zeppelin-python</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>net.sf.py4j</groupId>
+          <artifactId>py4j</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -95,12 +121,6 @@
       <scope>provided</scope>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${guava.version}</version>
-    </dependency>
-
     <!-- Aether :: maven dependency resolution -->
     <dependency>
       <groupId>org.apache.maven</groupId>
@@ -355,6 +375,12 @@
             <exclude>**/SparkRInterpreterTest.java</exclude>
             <exclude>${pyspark.test.exclude}</exclude>
           </excludes>
+          <environmentVariables>
+            <!-- local pyspark execution needs PYTHONPATH otherwise python daemon in executor side will fail
+            e.g. sc.range(1,10).sum()
+            -->
+            <PYTHONPATH>../interpreter/spark/pyspark/pyspark.zip:../interpreter/spark/pyspark/py4j-${spark.py4j.version}-src.zip:../interpreter/lib/python</PYTHONPATH>
+          </environmentVariables>
         </configuration>
       </plugin>
 
@@ -379,6 +405,19 @@
               <resource>reference.conf</resource>
             </transformer>
           </transformers>
+
+          <relocations>
+            <!-- shade guava and proto-buf, because it might conflict with those of spark -->
+            <relocation>
+              <pattern>com.google</pattern>
+              <shadedPattern>org.apache.zeppelin.com.google</shadedPattern>
+            </relocation>
+            <!-- shade netty, because it might conflict with that of spark-->
+            <relocation>
+              <pattern>io.netty</pattern>
+              <shadedPattern>org.apache.zeppelin.io.netty</shadedPattern>
+            </relocation>
+          </relocations>
         </configuration>
         <executions>
           <execution>

spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.spark;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
+import org.apache.zeppelin.interpreter.WrappedInterpreter;
+import org.apache.zeppelin.python.IPythonInterpreter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * PySparkInterpreter which use IPython underlying.
+ */
+public class IPySparkInterpreter extends IPythonInterpreter {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPySparkInterpreter.class);
+
+  private SparkInterpreter sparkInterpreter;
+
+  public IPySparkInterpreter(Properties property) {
+    super(property);
+  }
+
+  @Override
+  public void open() {
+    getProperty().setProperty("zeppelin.python", PySparkInterpreter.getPythonExec(property));
+    sparkInterpreter = getSparkInterpreter();
+    SparkConf conf = sparkInterpreter.getSparkContext().getConf();
+    String additionalPythonPath = conf.get("spark.submit.pyFiles").replaceAll(",", ":") +
+        ":../interpreter/lib/python";
+    setAdditionalPythonPath(additionalPythonPath);
+    setAdditionalPythonInitFile("python/zeppelin_ipyspark.py");
+    super.open();
+  }
+
+  private SparkInterpreter getSparkInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    SparkInterpreter spark = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    spark = (SparkInterpreter) p;
+
+    if (lazy != null) {
+      lazy.open();
+    }
+    return spark;
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+    super.cancel(context);
+    sparkInterpreter.cancel(context);
+  }
+
+  @Override
+  public void close() {
+    super.close();
+    if (sparkInterpreter != null) {
+      sparkInterpreter.close();
+    }
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return sparkInterpreter.getProgress(context);
+  }
+
+  public boolean isSpark2() {
+    return sparkInterpreter.getSparkVersion().newerThanEquals(SparkVersion.SPARK_2_0_0);
+  }
+
+  public JavaSparkContext getJavaSparkContext() {
+    return sparkInterpreter.getJavaSparkContext();
+  }
+
+  public Object getSQLContext() {
+    return sparkInterpreter.getSQLContext();
+  }
+
+  public Object getSparkSession() {
+    return sparkInterpreter.getSparkSession();
+  }
+}

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -76,6 +76,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
   private static final int MAX_TIMEOUT_SEC = 10;
   private long pythonPid;
 
+  private IPySparkInterpreter iPySparkInterpreter;
+
   public PySparkInterpreter(Properties property) {
     super(property);
 
@@ -111,6 +113,37 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
   @Override
   public void open() {
+    // try IPySparkInterpreter first
+    iPySparkInterpreter = getIPySparkInterpreter();
+    if (property.getProperty("zeppelin.spark.useIPython", "true").equals("true") &&
+        iPySparkInterpreter.checkIPythonPrerequisite()) {
+      try {
+        iPySparkInterpreter.open();
+        if (InterpreterContext.get() != null) {
+          // don't print it when it is in testing, just for easy output check in test.
+          InterpreterContext.get().out.write(("IPython is available, " +
+              "use IPython for PySparkInterpreter\n")
+              .getBytes());
+        }
+        LOGGER.info("Use IPySparkInterpreter to replace PySparkInterpreter");
+        return;
+      } catch (Exception e) {
+        LOGGER.warn("Fail to open IPySparkInterpreter", e);
+      }
+    }
+    iPySparkInterpreter = null;
+
+    if (property.getProperty("zeppelin.spark.useIPython", "true").equals("true")) {
+      // don't print it when it is in testing, just for easy output check in test.
+      try {
+        InterpreterContext.get().out.write(("IPython is not available, " +
+            "use the native PySparkInterpreter\n")
+            .getBytes());
+      } catch (IOException e) {
+        LOGGER.warn("Fail to write InterpreterOutput", e);
+      }
+    }
+
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
@@ -190,9 +223,24 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
       }
     }
 
+    LOGGER.debug("PYTHONPATH: " + env.get("PYTHONPATH"));
     return env;
   }
 
+  // Run python shell
+  // Choose python in the order of
+  // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
+  public static String getPythonExec(Properties properties) {
+    String pythonExec = properties.getProperty("zeppelin.pyspark.python", "python");
+    if (System.getenv("PYSPARK_PYTHON") != null) {
+      pythonExec = System.getenv("PYSPARK_PYTHON");
+    }
+    if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
+      pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
+    }
+    return pythonExec;
+  }
+
   private void createGatewayServerAndStartScript() {
     // create python script
     createPythonScript();
@@ -202,16 +250,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     gatewayServer = new GatewayServer(this, port);
     gatewayServer.start();
 
-    // Run python shell
-    // Choose python in the order of
-    // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
-    String pythonExec = getProperty("zeppelin.pyspark.python");
-    if (System.getenv("PYSPARK_PYTHON") != null) {
-      pythonExec = System.getenv("PYSPARK_PYTHON");
-    }
-    if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
-      pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
-    }
+    String pythonExec = getPythonExec(property);
     CommandLine cmd = CommandLine.parse(pythonExec);
     cmd.addArgument(scriptPath, false);
     cmd.addArgument(Integer.toString(port), false);
@@ -263,6 +302,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
   @Override
   public void close() {
+    if (iPySparkInterpreter != null) {
+      iPySparkInterpreter.close();
+      return;
+    }
     executor.getWatchdog().destroyProcess();
     new File(scriptPath).delete();
     gatewayServer.shutdown();
@@ -353,6 +396,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
           + sparkInterpreter.getSparkVersion().toString() + " is not supported");
     }
 
+    if (iPySparkInterpreter != null) {
+      return iPySparkInterpreter.interpret(st, context);
+    }
+
     if (!pythonscriptRunning) {
       return new InterpreterResult(Code.ERROR, "python process not running"
           + outputStream.toString());
@@ -448,6 +495,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
   @Override
   public void cancel(InterpreterContext context) {
+    if (iPySparkInterpreter != null) {
+      iPySparkInterpreter.cancel(context);
+      return;
+    }
     SparkInterpreter sparkInterpreter = getSparkInterpreter();
     sparkInterpreter.cancel(context);
     try {
@@ -464,6 +515,9 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
 
   @Override
   public int getProgress(InterpreterContext context) {
+    if (iPySparkInterpreter != null) {
+      return iPySparkInterpreter.getProgress(context);
+    }
     SparkInterpreter sparkInterpreter = getSparkInterpreter();
     return sparkInterpreter.getProgress(context);
   }
@@ -472,6 +526,9 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
   @Override
   public List<InterpreterCompletion> completion(String buf, int cursor,
       InterpreterContext interpreterContext) {
+    if (iPySparkInterpreter != null) {
+      return iPySparkInterpreter.completion(buf, cursor, interpreterContext);
+    }
     if (buf.length() < cursor) {
       cursor = buf.length();
     }
@@ -588,6 +645,21 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     return spark;
   }
 
+  private IPySparkInterpreter getIPySparkInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    IPySparkInterpreter iPySpark = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(IPySparkInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    iPySpark = (IPySparkInterpreter) p;
+    return iPySpark;
+  }
+
   public SparkZeppelinContext getZeppelinContext() {
     SparkInterpreter sparkIntp = getSparkInterpreter();
     if (sparkIntp != null) {

spark/src/main/resources/interpreter-setting.json

@@ -149,11 +149,28 @@
         "defaultValue": "python",
         "description": "Python command to run pyspark with",
         "type": "string"
+      },
+      "zeppelin.spark.useIPython": {
+        "envName": null,
+        "propertyName": "zeppelin.spark.useIPython",
+        "defaultValue": true,
+        "description": "whether use IPython when it is available",
+        "type": "checkbox"
       }
     },
     "editor": {
       "language": "python",
       "editOnDblClick": false
     }
+  },
+  {
+    "group": "spark",
+    "name": "ipyspark",
+    "className": "org.apache.zeppelin.spark.IPySparkInterpreter",
+    "properties": {},
+    "editor": {
+      "language": "python",
+      "editOnDblClick": false
+    }
   }
 ]

spark/src/main/resources/python/zeppelin_ipyspark.py

@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from py4j.java_gateway import java_import, JavaGateway, GatewayClient
+from pyspark.conf import SparkConf
+from pyspark.context import SparkContext
+
+# for back compatibility
+from pyspark.sql import SQLContext
+
+# start JVM gateway
+client = GatewayClient(port=${JVM_GATEWAY_PORT})
+gateway = JavaGateway(client, auto_convert=True)
+
+java_import(gateway.jvm, "org.apache.spark.SparkEnv")
+java_import(gateway.jvm, "org.apache.spark.SparkConf")
+java_import(gateway.jvm, "org.apache.spark.api.java.*")
+java_import(gateway.jvm, "org.apache.spark.api.python.*")
+java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
+
+intp = gateway.entry_point
+jsc = intp.getJavaSparkContext()
+
+java_import(gateway.jvm, "org.apache.spark.sql.*")
+java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
+java_import(gateway.jvm, "scala.Tuple2")
+
+jconf = jsc.getConf()
+conf = SparkConf(_jvm=gateway.jvm, _jconf=jconf)
+sc = _zsc_ = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
+
+if intp.isSpark2():
+    from pyspark.sql import SparkSession
+
+    spark = __zSpark__ = SparkSession(sc, intp.getSparkSession())
+    sqlContext = sqlc = __zSqlc__ = __zSpark__._wrapped
+else:
+    sqlContext = sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())

spark/src/main/sparkr-resources/interpreter-setting.json

@@ -189,5 +189,16 @@
     "editor": {
       "language": "r"
     }
+  },
+
+  {
+    "group": "spark",
+    "name": "ipyspark",
+    "className": "org.apache.zeppelin.spark.IPySparkInterpreter",
+    "properties": {},
+    "editor": {
+      "language": "python",
+      "editOnDblClick": false
+    }
   }
 ]

spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java

@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.spark;
+
+
+import com.google.common.io.Files;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterOutputListener;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.python.IPythonInterpreterTest;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class IPySparkInterpreterTest {
+
+  private IPySparkInterpreter iPySparkInterpreter;
+  private InterpreterGroup intpGroup;
+
+  @Before
+  public void setup() {
+    Properties p = new Properties();
+    p.setProperty("spark.master", "local[4]");
+    p.setProperty("master", "local[4]");
+    p.setProperty("spark.app.name", "Zeppelin Test");
+    p.setProperty("zeppelin.spark.useHiveContext", "true");
+    p.setProperty("zeppelin.spark.maxResult", "1000");
+    p.setProperty("zeppelin.spark.importImplicit", "true");
+    p.setProperty("zeppelin.pyspark.python", "python");
+    p.setProperty("zeppelin.dep.localrepo", Files.createTempDir().getAbsolutePath());
+
+    intpGroup = new InterpreterGroup();
+    intpGroup.put("session_1", new LinkedList<Interpreter>());
+
+    SparkInterpreter sparkInterpreter = new SparkInterpreter(p);
+    intpGroup.get("session_1").add(sparkInterpreter);
+    sparkInterpreter.setInterpreterGroup(intpGroup);
+    sparkInterpreter.open();
+
+    iPySparkInterpreter = new IPySparkInterpreter(p);
+    intpGroup.get("session_1").add(iPySparkInterpreter);
+    iPySparkInterpreter.setInterpreterGroup(intpGroup);
+    iPySparkInterpreter.open();
+  }
+
+
+  @After
+  public void tearDown() {
+    if (iPySparkInterpreter != null) {
+      iPySparkInterpreter.close();
+    }
+  }
+
+  @Test
+  public void testBasics() throws InterruptedException, IOException {
+    // all the ipython test should pass too.
+    IPythonInterpreterTest.testInterpreter(iPySparkInterpreter);
+
+    // rdd
+    InterpreterContext context = getInterpreterContext();
+    InterpreterResult result = iPySparkInterpreter.interpret("sc.range(1,10).sum()", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    List<InterpreterResultMessage> interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals("45", interpreterResultMessages.get(0).getData());
+
+    context = getInterpreterContext();
+    result = iPySparkInterpreter.interpret("sc.version", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    // spark sql
+    context = getInterpreterContext();
+    if (interpreterResultMessages.get(0).getData().startsWith("'1.") ||
+        interpreterResultMessages.get(0).getData().startsWith("u'1.")) {
+      result = iPySparkInterpreter.interpret("df = sqlContext.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      interpreterResultMessages = context.out.getInterpreterResultMessages();
+      assertEquals(
+          "+---+---+\n" +
+          "| _1| _2|\n" +
+          "+---+---+\n" +
+          "|  1|  a|\n" +
+          "|  2|  b|\n" +
+          "+---+---+\n\n", interpreterResultMessages.get(0).getData());
+    } else {
+      result = iPySparkInterpreter.interpret("df = spark.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      interpreterResultMessages = context.out.getInterpreterResultMessages();
+      assertEquals(
+          "+---+---+\n" +
+          "| _1| _2|\n" +
+          "+---+---+\n" +
+          "|  1|  a|\n" +
+          "|  2|  b|\n" +
+          "+---+---+\n\n", interpreterResultMessages.get(0).getData());
+    }
+
+    // cancel
+    final InterpreterContext context2 = getInterpreterContext();
+
+    Thread thread = new Thread(){
+      @Override
+      public void run() {
+        InterpreterResult result = iPySparkInterpreter.interpret("import time\nsc.range(1,10).foreach(lambda x: time.sleep(1))", context2);
+        assertEquals(InterpreterResult.Code.ERROR, result.code());
+        List<InterpreterResultMessage> interpreterResultMessages = null;
+        try {
+          interpreterResultMessages = context2.out.getInterpreterResultMessages();
+          assertTrue(interpreterResultMessages.get(0).getData().contains("cancelled"));
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+    };
+    thread.start();
+
+    // sleep 1 second to wait for the spark job starts
+    Thread.sleep(1000);
+    iPySparkInterpreter.cancel(context);
+    thread.join();
+
+    // completions
+    List<InterpreterCompletion> completions = iPySparkInterpreter.completion("sc.ran", 6, getInterpreterContext());
+    assertEquals(1, completions.size());
+    assertEquals("sc.range", completions.get(0).getValue());
+
+    // pyspark streaming
+    context = getInterpreterContext();
+    result = iPySparkInterpreter.interpret(
+        "from pyspark.streaming import StreamingContext\n" +
+        "import time\n" +
+        "ssc = StreamingContext(sc, 1)\n" +
+        "rddQueue = []\n" +
+        "for i in range(5):\n" +
+        "    rddQueue += [ssc.sparkContext.parallelize([j for j in range(1, 1001)], 10)]\n" +
+        "inputStream = ssc.queueStream(rddQueue)\n" +
+        "mappedStream = inputStream.map(lambda x: (x % 10, 1))\n" +
+        "reducedStream = mappedStream.reduceByKey(lambda a, b: a + b)\n" +
+        "reducedStream.pprint()\n" +
+        "ssc.start()\n" +
+        "time.sleep(6)\n" +
+        "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context);
+    Thread.sleep(100);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.getInterpreterResultMessages();
+    assertEquals(1, interpreterResultMessages.size());
+    assertTrue(interpreterResultMessages.get(0).getData().contains("(0, 100)"));
+  }
+
+  private InterpreterContext getInterpreterContext() {
+    return new InterpreterContext(
+        "noteId",
+        "paragraphId",
+        "replName",
+        "paragraphTitle",
+        "paragraphText",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        null,
+        null,
+        null,
+        new InterpreterOutput(null));
+  }
+}

spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java

@@ -89,6 +89,7 @@ public class PySparkInterpreterMatplotlibTest {
     p.setProperty("zeppelin.spark.importImplicit", "true");
     p.setProperty("zeppelin.pyspark.python", "python");
     p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
+    p.setProperty("zeppelin.spark.useIPython", "false");
     return p;
   }
 
@@ -110,6 +111,15 @@ public class PySparkInterpreterMatplotlibTest {
   public static void setUp() throws Exception {
     intpGroup = new InterpreterGroup();
     intpGroup.put("note", new LinkedList<Interpreter>());
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        new InterpreterOutput(null));
+    InterpreterContext.set(context);
 
     sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
     intpGroup.get("note").add(sparkInterpreter);
@@ -121,14 +131,6 @@ public class PySparkInterpreterMatplotlibTest {
     pyspark.setInterpreterGroup(intpGroup);
     pyspark.open();
 
-    context = new InterpreterContext("note", "id", null, "title", "text",
-      new AuthenticationInfo(),
-      new HashMap<String, Object>(),
-      new GUI(),
-      new AngularObjectRegistry(intpGroup.getId(), null),
-      new LocalResourcePool("id"),
-      new LinkedList<InterpreterContextRunner>(),
-      new InterpreterOutput(null));
   }
 
   @AfterClass

spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java

@@ -59,6 +59,7 @@ public class PySparkInterpreterTest {
     p.setProperty("zeppelin.spark.importImplicit", "true");
     p.setProperty("zeppelin.pyspark.python", "python");
     p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
+    p.setProperty("zeppelin.spark.useIPython", "false");
     return p;
   }
 
@@ -81,6 +82,16 @@ public class PySparkInterpreterTest {
     intpGroup = new InterpreterGroup();
     intpGroup.put("note", new LinkedList<Interpreter>());
 
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        new InterpreterOutput(null));
+    InterpreterContext.set(context);
+
     sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
     intpGroup.get("note").add(sparkInterpreter);
     sparkInterpreter.setInterpreterGroup(intpGroup);
@@ -91,14 +102,7 @@ public class PySparkInterpreterTest {
     pySparkInterpreter.setInterpreterGroup(intpGroup);
     pySparkInterpreter.open();
 
-    context = new InterpreterContext("note", "id", null, "title", "text",
-      new AuthenticationInfo(),
-      new HashMap<String, Object>(),
-      new GUI(),
-      new AngularObjectRegistry(intpGroup.getId(), null),
-      new LocalResourcePool("id"),
-      new LinkedList<InterpreterContextRunner>(),
-      new InterpreterOutput(null));
+
   }
 
   @AfterClass
@@ -113,6 +117,22 @@ public class PySparkInterpreterTest {
       assertEquals(InterpreterResult.Code.SUCCESS,
         pySparkInterpreter.interpret("a = 1\n", context).code());
     }
+
+    InterpreterResult result = pySparkInterpreter.interpret(
+        "from pyspark.streaming import StreamingContext\n" +
+            "import time\n" +
+            "ssc = StreamingContext(sc, 1)\n" +
+            "rddQueue = []\n" +
+            "for i in range(5):\n" +
+            "    rddQueue += [ssc.sparkContext.parallelize([j for j in range(1, 1001)], 10)]\n" +
+            "inputStream = ssc.queueStream(rddQueue)\n" +
+            "mappedStream = inputStream.map(lambda x: (x % 10, 1))\n" +
+            "reducedStream = mappedStream.reduceByKey(lambda a, b: a + b)\n" +
+            "reducedStream.pprint()\n" +
+            "ssc.start()\n" +
+            "time.sleep(6)\n" +
+            "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
   }
 
   @Test

spark/src/test/resources/log4j.properties

@@ -45,3 +45,5 @@ log4j.logger.org.hibernate.type=ALL
 
 log4j.logger.org.apache.zeppelin.interpreter=DEBUG
 log4j.logger.org.apache.zeppelin.spark=DEBUG
+log4j.logger.org.apache.zeppelin.python.IPythonInterpreter=DEBUG
+log4j.logger.org.apache.zeppelin.python.IPythonClient=DEBUG

testing/downloadLivy.sh

@@ -49,7 +49,7 @@ download_with_retry() {
 }
 
 LIVY_CACHE=".livy-dist"
-LIVY_ARCHIVE="livy-assembly-${LIVY_VERSION}"
+LIVY_ARCHIVE="livy-${LIVY_VERSION}-bin"
 export LIVY_HOME="${ZEPPELIN_HOME}/livy-server-$LIVY_VERSION"
 echo "LIVY_HOME is ${LIVY_HOME}"
 
@@ -64,7 +64,7 @@ if [[ ! -d "${LIVY_HOME}" ]]; then
         # download livy from archive if not cached
         echo "${LIVY_VERSION} being downloaded from archives"
         STARTTIME=`date +%s`
-        download_with_retry "https://oss.sonatype.org/content/repositories/releases/com/cloudera/livy/livy-assembly/${LIVY_VERSION}/${LIVY_ARCHIVE}.zip"
+        download_with_retry "https://dist.apache.org/repos/dist/release/incubator/livy/${LIVY_VERSION}/${LIVY_ARCHIVE}.zip"
         ENDTIME=`date +%s`
         DOWNLOADTIME="$((ENDTIME-STARTTIME))"
     fi

testing/install_external_dependencies.sh

@@ -44,5 +44,6 @@ if [[ -n "$PYTHON" ]] ; then
   conda update -q conda
   conda info -a
   conda config --add channels conda-forge
-  conda install -q matplotlib pandasql
+  conda install -q matplotlib pandasql ipython jupyter_client ipykernel matplotlib bokeh
+  pip install grpcio ggplot
 fi

travis_check.py

@@ -57,7 +57,7 @@ def getBuildStatus(author, commit):
     build = None
 
     if len(data) == 0:
-        return build;
+        return build
 
     for b in data:
         if b["commit"][:len(commit)] == commit:
@@ -102,7 +102,7 @@ def printBuildStatus(build):
 
 for sleep in check:
     info("--------------------------------")
-    time.sleep(sleep);
+    time.sleep(sleep)
     info("Get build status ...")
     build = getBuildStatus(author, commit)
     if build == None:

zeppelin-distribution/src/bin_license/LICENSE

@@ -272,6 +272,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (The MIT License) ngclipboard v1.1.1 (https://github.com/sachinchoolur/ngclipboard) - https://github.com/sachinchoolur/ngclipboard/blob/1.1.1/LICENSE
     (The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE
     (The MIT License) angular-viewport-watch 0.135 (https://github.com/wix/angular-viewport-watch) - https://github.com/wix/angular-viewport-watch/blob/master/LICENSE
+    (The MIT License) ansi-up 2.0.2 (https://github.com/drudru/ansi_up) - https://github.com/drudru/ansi_up#license
 
 ========================================================================
 BSD-style licenses

zeppelin-interpreter/pom.xml

@@ -214,11 +214,6 @@
       <version>${jline.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java

@@ -19,6 +19,8 @@ package org.apache.zeppelin.dep;
 
 import org.apache.commons.lang.Validate;
 import org.apache.maven.repository.internal.MavenRepositorySystemSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonatype.aether.RepositorySystem;
 import org.sonatype.aether.RepositorySystemSession;
 import org.sonatype.aether.repository.LocalRepository;
@@ -30,6 +32,8 @@ import java.nio.file.Paths;
  * Manage mvn repository.
  */
 public class Booter {
+  private static Logger logger = LoggerFactory.getLogger(Booter.class);
+
   public static RepositorySystem newRepositorySystem() {
     return RepositorySystemFactory.newRepositorySystem();
   }
@@ -43,9 +47,10 @@ public class Booter {
     LocalRepository localRepo = new LocalRepository(resolveLocalRepoPath(localRepoPath));
     session.setLocalRepositoryManager(system.newLocalRepositoryManager(localRepo));
 
-    // session.setTransferListener(new ConsoleTransferListener());
-    // session.setRepositoryListener(new ConsoleRepositoryListener());
-
+    if (logger.isDebugEnabled()) {
+      session.setTransferListener(new TransferListener());
+      session.setRepositoryListener(new RepositoryListener());
+    }
     // uncomment to generate dirty trees
     // session.setDependencyGraphTransformer( null );
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java

@@ -19,7 +19,6 @@ package org.apache.zeppelin.dep;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URL;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Iterator;
@@ -33,25 +32,23 @@ import org.slf4j.LoggerFactory;
 import org.sonatype.aether.RepositoryException;
 import org.sonatype.aether.artifact.Artifact;
 import org.sonatype.aether.collection.CollectRequest;
-import org.sonatype.aether.collection.DependencyCollectionException;
 import org.sonatype.aether.graph.Dependency;
 import org.sonatype.aether.graph.DependencyFilter;
 import org.sonatype.aether.repository.RemoteRepository;
 import org.sonatype.aether.resolution.ArtifactResult;
 import org.sonatype.aether.resolution.DependencyRequest;
+import org.sonatype.aether.resolution.DependencyResolutionException;
 import org.sonatype.aether.util.artifact.DefaultArtifact;
 import org.sonatype.aether.util.artifact.JavaScopes;
 import org.sonatype.aether.util.filter.DependencyFilterUtils;
 import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
-import org.sonatype.aether.util.graph.DefaultDependencyNode;
-
 
 /**
  * Deps resolver.
  * Add new dependencies from mvn repo (at runtime) to Zeppelin.
  */
 public class DependencyResolver extends AbstractDependencyResolver {
-  Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
+  private Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
 
   private final String[] exclusions = new String[] {"org.apache.zeppelin:zeppelin-zengine",
                                                     "org.apache.zeppelin:zeppelin-interpreter",
@@ -177,8 +174,9 @@ public class DependencyResolver extends AbstractDependencyResolver {
             DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
     try {
       return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
-    } catch (NullPointerException ex) {
-      throw new RepositoryException(String.format("Cannot fetch dependencies for %s", dependency));
+    } catch (NullPointerException | DependencyResolutionException ex) {
+      throw new RepositoryException(
+              String.format("Cannot fetch dependencies for %s", dependency), ex);
     }
   }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java

@@ -17,7 +17,6 @@
 
 package org.apache.zeppelin.dep;
 
-import java.io.PrintStream;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.util.Locale;
@@ -34,8 +33,7 @@ import org.sonatype.aether.transfer.TransferResource;
  * Simple listener that show deps downloading progress.
  */
 public class TransferListener extends AbstractTransferListener {
-  Logger logger = LoggerFactory.getLogger(TransferListener.class);
-  private PrintStream out;
+  private Logger logger = LoggerFactory.getLogger(TransferListener.class);
 
   private Map<TransferResource, Long> downloads = new ConcurrentHashMap<>();
 
@@ -55,13 +53,13 @@ public class TransferListener extends AbstractTransferListener {
   @Override
   public void transferProgressed(TransferEvent event) {
     TransferResource resource = event.getResource();
-    downloads.put(resource, Long.valueOf(event.getTransferredBytes()));
+    downloads.put(resource, event.getTransferredBytes());
 
     StringBuilder buffer = new StringBuilder(64);
 
     for (Map.Entry<TransferResource, Long> entry : downloads.entrySet()) {
       long total = entry.getKey().getContentLength();
-      long complete = entry.getValue().longValue();
+      long complete = entry.getValue();
 
       buffer.append(getStatus(complete, total)).append("  ");
     }
@@ -122,7 +120,7 @@ public class TransferListener extends AbstractTransferListener {
   @Override
   public void transferFailed(TransferEvent event) {
     transferCompleted(event);
-    event.getException().printStackTrace(out);
+    logger.warn("Unsuccessful transfer", event.getException());
   }
 
   private void transferCompleted(TransferEvent event) {
@@ -135,10 +133,10 @@ public class TransferListener extends AbstractTransferListener {
 
   @Override
   public void transferCorrupted(TransferEvent event) {
-    event.getException().printStackTrace(out);
+    logger.error("Corrupted transfer", event.getException());
   }
 
-  protected long toKB(long bytes) {
+  private long toKB(long bytes) {
     return (bytes + 1023) / 1024;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java

@@ -123,6 +123,10 @@ public abstract class BaseZeppelinContext {
     this.gui = o;
   }
 
+  public GUI getGui() {
+    return gui;
+  }
+
   private void restartInterpreter() {
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -149,7 +149,6 @@ public abstract class Interpreter {
 
   @ZeppelinApi
   public Interpreter(Properties property) {
-    logger.debug("Properties: {}", property);
     this.property = property;
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java

@@ -17,107 +17,90 @@
 
 package org.apache.zeppelin.interpreter;
 
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
 import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.scheduler.Scheduler;
-import org.apache.zeppelin.scheduler.SchedulerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
 /**
- * InterpreterGroup is list of interpreters in the same interpreter group.
- * For example spark, pyspark, sql interpreters are in the same 'spark' group
- * and InterpreterGroup will have reference to these all interpreters.
+ * InterpreterGroup is collections of interpreter sessions.
+ * One session could include multiple interpreters.
+ * For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
  *
  * Remember, list of interpreters are dedicated to a session. Session could be shared across user
  * or notes, so the sessionId could be user or noteId or their combination.
- * So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
+ * So InterpreterGroup internally manages map of [sessionId(noteId, user, or
  * their combination), list of interpreters]
  *
- * A InterpreterGroup runs on interpreter process.
- * And unit of interpreter instantiate, restart, bind, unbind.
+ * A InterpreterGroup runs interpreter process while its subclass ManagedInterpreterGroup runs
+ * in zeppelin server process.
  */
-public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
-  String id;
+public class InterpreterGroup {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
 
-  AngularObjectRegistry angularObjectRegistry;
-  InterpreterHookRegistry hookRegistry;
-  RemoteInterpreterProcess remoteInterpreterProcess;    // attached remote interpreter process
-  ResourcePool resourcePool;
-  boolean angularRegistryPushed = false;
-
-  // map [notebook session, Interpreters in the group], to support per note session interpreters
-  //Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
-  // List<Interpreter>>();
-
-  private static final Map<String, InterpreterGroup> allInterpreterGroups =
-      new ConcurrentHashMap<>();
-
-  public static InterpreterGroup getByInterpreterGroupId(String id) {
-    return allInterpreterGroups.get(id);
-  }
-
-  public static Collection<InterpreterGroup> getAll() {
-    return new LinkedList(allInterpreterGroups.values());
-  }
+  protected String id;
+  // sessionId --> interpreters
+  protected Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
+  private AngularObjectRegistry angularObjectRegistry;
+  private InterpreterHookRegistry hookRegistry;
+  private ResourcePool resourcePool;
+  private boolean angularRegistryPushed = false;
 
   /**
-   * Create InterpreterGroup with given id
+   * Create InterpreterGroup with given id, used in InterpreterProcess
    * @param id
    */
   public InterpreterGroup(String id) {
     this.id = id;
-    allInterpreterGroups.put(id, this);
   }
 
   /**
    * Create InterpreterGroup with autogenerated id
    */
   public InterpreterGroup() {
-    getId();
-    allInterpreterGroups.put(id, this);
+    this.id = generateId();
   }
 
   private static String generateId() {
-    return "InterpreterGroup_" + System.currentTimeMillis() + "_"
-           + new Random().nextInt();
+    return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
   }
 
   public String getId() {
-    synchronized (this) {
-      if (id == null) {
-        id = generateId();
-      }
-      return id;
-    }
+    return this.id;
   }
 
-  /**
-   * Get combined property of all interpreters in this group
-   * @return
-   */
-  public Properties getProperty() {
-    Properties p = new Properties();
+  //TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
+  public synchronized List<Interpreter> get(String sessionId) {
+    return sessions.get(sessionId);
+  }
 
-    for (List<Interpreter> intpGroupForASession : this.values()) {
-      for (Interpreter intp : intpGroupForASession) {
-        p.putAll(intp.getProperty());
-      }
-      // it's okay to break here while every List<Interpreters> will have the same property set
-      break;
+  //TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
+  public synchronized void put(String sessionId, List<Interpreter> interpreters) {
+    this.sessions.put(sessionId, interpreters);
+  }
+
+  public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
+    LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
+    List<Interpreter> interpreters = get(sessionId);
+    if (interpreters == null) {
+      interpreters = new ArrayList<>();
     }
-    return p;
+    interpreters.add(interpreter);
+    put(sessionId, interpreters);
+  }
+
+  //TODO(zjffdu) rename it to a more proper name.
+  //For now just keep this method to reduce code change
+  public Collection<List<Interpreter>> values() {
+    return sessions.values();
   }
 
   public AngularObjectRegistry getAngularObjectRegistry() {
@@ -136,128 +119,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
     this.hookRegistry = hookRegistry;
   }
 
-  public RemoteInterpreterProcess getRemoteInterpreterProcess() {
-    return remoteInterpreterProcess;
-  }
-
-  public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
-    this.remoteInterpreterProcess = remoteInterpreterProcess;
-  }
-
-  /**
-   * Close all interpreter instances in this group
-   */
-  public void close() {
-    LOGGER.info("Close interpreter group " + getId());
-    List<Interpreter> intpToClose = new LinkedList<>();
-    for (List<Interpreter> intpGroupForSession : this.values()) {
-      intpToClose.addAll(intpGroupForSession);
-    }
-    close(intpToClose);
-
-    // make sure remote interpreter process terminates
-    if (remoteInterpreterProcess != null) {
-      while (remoteInterpreterProcess.referenceCount() > 0) {
-        remoteInterpreterProcess.dereference();
-      }
-      remoteInterpreterProcess = null;
-    }
-    allInterpreterGroups.remove(id);
-  }
-
-  /**
-   * Close all interpreter instances in this group for the session
-   * @param sessionId
-   */
-  public void close(String sessionId) {
-    LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
-    final List<Interpreter> intpForSession = this.get(sessionId);
-
-    close(intpForSession);
-  }
-
-  private void close(final Collection<Interpreter> intpToClose) {
-    close(null, null, null, intpToClose);
-  }
-
-  public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
-      final String processKey, final String sessionKey) {
-    LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
-    close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
-  }
-
-  private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
-      final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
-    if (intpToClose == null) {
-      return;
-    }
-    Thread t = new Thread() {
-      public void run() {
-        for (Interpreter interpreter : intpToClose) {
-          Scheduler scheduler = interpreter.getScheduler();
-          interpreter.close();
-
-          if (null != scheduler) {
-            SchedulerFactory.singleton().removeScheduler(scheduler.getName());
-          }
-        }
-
-        if (remoteInterpreterProcess != null) {
-          //TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
-          // refernceCount is a proper value. And as the same reason, we must not call
-          // remoteInterpreterProcess.dereference twice - this method also be called by
-          // interpreter.close().
-
-          // remoteInterpreterProcess.dereference();
-          if (remoteInterpreterProcess.referenceCount() <= 0) {
-            remoteInterpreterProcess = null;
-            allInterpreterGroups.remove(id);
-          }
-        }
-
-        // TODO(jl): While closing interpreters in a same session, we should remove after all
-        // interpreters are removed. OMG. It's too dirty!!
-        if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
-          InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
-          if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
-            interpreterGroupRef.remove(processKey);
-          } else {
-            interpreterGroup.remove(sessionKey);
-          }
-        }
-      }
-    };
-
-    t.start();
-    try {
-      t.join();
-    } catch (InterruptedException e) {
-      LOGGER.error("Can't close interpreter: {}", getId(), e);
-    }
-
-
-  }
-
-  /**
-   * Close all interpreter instances in this group
-   */
-  public void shutdown() {
-    LOGGER.info("Close interpreter group " + getId());
-
-    // make sure remote interpreter process terminates
-    if (remoteInterpreterProcess != null) {
-      while (remoteInterpreterProcess.referenceCount() > 0) {
-        remoteInterpreterProcess.dereference();
-      }
-      remoteInterpreterProcess = null;
-    }
-    allInterpreterGroups.remove(id);
-
-    List<Interpreter> intpToClose = new LinkedList<>();
-    for (List<Interpreter> intpGroupForSession : this.values()) {
-      intpToClose.addAll(intpGroupForSession);
-    }
-    close(intpToClose);
+  public int getSessionNum() {
+    return sessions.size();
   }
 
   public void setResourcePool(ResourcePool resourcePool) {
@@ -275,4 +138,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
   public void setAngularRegistryPushed(boolean angularRegistryPushed) {
     this.angularRegistryPushed = angularRegistryPushed;
   }
+
+  public boolean isEmpty() {
+    return sessions.isEmpty();
+  }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java

@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
 import java.io.*;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -115,6 +116,16 @@ public class InterpreterOutput extends OutputStream {
     };
   }
 
+  public List<InterpreterResultMessage> getInterpreterResultMessages() throws IOException {
+    synchronized (resultMessageOutputs) {
+      List<InterpreterResultMessage> resultMessages = new ArrayList<>();
+      for (InterpreterResultMessageOutput output : this.resultMessageOutputs) {
+        resultMessages.add(output.toInterpreterResultMessage());
+      }
+      return resultMessages;
+    }
+  }
+
   public InterpreterResultMessageOutput getCurrentOutput() {
     synchronized (resultMessageOutputs) {
       return currentOut;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java

@@ -12,6 +12,15 @@ public class InterpreterRunner {
   @SerializedName("win")
   private String winPath;
 
+  public InterpreterRunner() {
+
+  }
+
+  public InterpreterRunner(String linuxPath, String winPath) {
+    this.linuxPath = linuxPath;
+    this.winPath = winPath;
+  }
+
   public String getPath() {
     return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -106,9 +106,14 @@ public class RemoteInterpreterServer
 
   @Override
   public void shutdown() throws TException {
+    logger.info("Shutting down...");
     eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
     if (interpreterGroup != null) {
-      interpreterGroup.close();
+      for (List<Interpreter> session : interpreterGroup.values()) {
+        for (Interpreter interpreter : session) {
+          interpreter.close();
+        }
+      }
     }
 
     server.stop();
@@ -159,7 +164,7 @@ public class RemoteInterpreterServer
   }
 
   @Override
-  public void createInterpreter(String interpreterGroupId, String sessionKey, String
+  public void createInterpreter(String interpreterGroupId, String sessionId, String
       className, Map<String, String> properties, String userName) throws TException {
     if (interpreterGroup == null) {
       interpreterGroup = new InterpreterGroup(interpreterGroupId);
@@ -190,20 +195,11 @@ public class RemoteInterpreterServer
           replClass.getConstructor(new Class[] {Properties.class});
       Interpreter repl = constructor.newInstance(p);
       repl.setClassloaderUrls(new URL[]{});
-
-      synchronized (interpreterGroup) {
-        List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
-        if (interpreters == null) {
-          interpreters = new LinkedList<>();
-          interpreterGroup.put(sessionKey, interpreters);
-        }
-
-        interpreters.add(new LazyOpenInterpreter(repl));
-      }
-
       logger.info("Instantiate interpreter {}", className);
       repl.setInterpreterGroup(interpreterGroup);
       repl.setUserName(userName);
+
+      interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId);
     } catch (ClassNotFoundException | NoSuchMethodException | SecurityException
         | InstantiationException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException e) {
@@ -237,13 +233,13 @@ public class RemoteInterpreterServer
     }
   }
 
-  protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+  protected Interpreter getInterpreter(String sessionId, String className) throws TException {
     if (interpreterGroup == null) {
       throw new TException(
           new InterpreterException("Interpreter instance " + className + " not created"));
     }
     synchronized (interpreterGroup) {
-      List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+      List<Interpreter> interpreters = interpreterGroup.get(sessionId);
       if (interpreters == null) {
         throw new TException(
             new InterpreterException("Interpreter " + className + " not initialized"));
@@ -259,19 +255,20 @@ public class RemoteInterpreterServer
   }
 
   @Override
-  public void open(String noteId, String className) throws TException {
-    Interpreter intp = getInterpreter(noteId, className);
+  public void open(String sessionId, String className) throws TException {
+    logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId));
+    Interpreter intp = getInterpreter(sessionId, className);
     intp.open();
   }
 
   @Override
-  public void close(String sessionKey, String className) throws TException {
+  public void close(String sessionId, String className) throws TException {
     // unload all applications
     for (String appId : runningApplications.keySet()) {
       RunningApplication appInfo = runningApplications.get(appId);
 
       // see NoteInterpreterLoader.SHARED_SESSION
-      if (appInfo.noteId.equals(sessionKey) || sessionKey.equals("shared_session")) {
+      if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) {
         try {
           logger.info("Unload App {} ", appInfo.pkg.getName());
           appInfo.app.unload();
@@ -286,7 +283,7 @@ public class RemoteInterpreterServer
     // close interpreters
     List<Interpreter> interpreters;
     synchronized (interpreterGroup) {
-      interpreters = interpreterGroup.get(sessionKey);
+      interpreters = interpreterGroup.get(sessionId);
     }
     if (interpreters != null) {
       Iterator<Interpreter> it = interpreters.iterator();
@@ -322,7 +319,6 @@ public class RemoteInterpreterServer
         intp,
         st,
         context);
-
     scheduler.submit(job);
 
     while (!job.isTerminated()) {
@@ -513,6 +509,13 @@ public class RemoteInterpreterServer
         List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
         resultMessages.addAll(result.message());
 
+        for (InterpreterResultMessage msg : resultMessages) {
+          if (msg.getType() == InterpreterResult.Type.IMG) {
+            logger.debug("InterpreterResultMessage: IMAGE_DATA");
+          } else {
+            logger.debug("InterpreterResultMessage: " + msg.toString());
+          }
+        }
         // put result into resource pool
         if (resultMessages.size() > 0) {
           int lastMessageIndex = resultMessages.size() - 1;
@@ -559,30 +562,34 @@ public class RemoteInterpreterServer
   }
 
   @Override
-  public int getProgress(String noteId, String className,
+  public int getProgress(String sessionId, String className,
                          RemoteInterpreterContext interpreterContext)
       throws TException {
     Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
     if (manuallyProvidedProgress != null) {
       return manuallyProvidedProgress;
     } else {
-      Interpreter intp = getInterpreter(noteId, className);
+      Interpreter intp = getInterpreter(sessionId, className);
+      if (intp == null) {
+        throw new TException("No interpreter {} existed for session {}".format(
+            className, sessionId));
+      }
       return intp.getProgress(convert(interpreterContext, null));
     }
   }
 
 
   @Override
-  public String getFormType(String noteId, String className) throws TException {
-    Interpreter intp = getInterpreter(noteId, className);
+  public String getFormType(String sessionId, String className) throws TException {
+    Interpreter intp = getInterpreter(sessionId, className);
     return intp.getFormType().toString();
   }
 
   @Override
-  public List<InterpreterCompletion> completion(String noteId,
+  public List<InterpreterCompletion> completion(String sessionId,
       String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
       throws TException {
-    Interpreter intp = getInterpreter(noteId, className);
+    Interpreter intp = getInterpreter(sessionId, className);
     List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
     return completion;
   }
@@ -759,16 +766,16 @@ public class RemoteInterpreterServer
   }
 
   @Override
-  public String getStatus(String sessionKey, String jobId)
+  public String getStatus(String sessionId, String jobId)
       throws TException {
     if (interpreterGroup == null) {
-      return "Unknown";
+      return Status.UNKNOWN.name();
     }
 
     synchronized (interpreterGroup) {
-      List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+      List<Interpreter> interpreters = interpreterGroup.get(sessionId);
       if (interpreters == null) {
-        return "Unknown";
+        return Status.UNKNOWN.name();
       }
 
       for (Interpreter intp : interpreters) {
@@ -785,7 +792,7 @@ public class RemoteInterpreterServer
         }
       }
     }
-    return "Unknown";
+    return Status.UNKNOWN.name();
   }
 
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java

@@ -29,7 +29,7 @@ import java.io.IOException;
  */
 public class InterpreterOutputStream extends LogOutputStream {
   private Logger logger;
-  InterpreterOutput interpreterOutput;
+  volatile InterpreterOutput interpreterOutput;
   boolean ignoreLeadingNewLinesFromScalaReporter = false;
 
   public InterpreterOutputStream(Logger logger) {

zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java

@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.resource;
-
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.slf4j.Logger;
-
-import java.util.List;
-
-/**
- * Utilities for ResourcePool
- */
-public class ResourcePoolUtils {
-  static Logger logger = org.slf4j.LoggerFactory.getLogger(ResourcePoolUtils.class);
-
-  public static ResourceSet getAllResources() {
-    return getAllResourcesExcept(null);
-  }
-
-  public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
-    ResourceSet resourceSet = new ResourceSet();
-    for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
-      if (interpreterGroupExcludsion != null &&
-          intpGroup.getId().equals(interpreterGroupExcludsion)) {
-        continue;
-      }
-
-      RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
-      if (remoteInterpreterProcess == null) {
-        ResourcePool localPool = intpGroup.getResourcePool();
-        if (localPool != null) {
-          resourceSet.addAll(localPool.getAll());
-        }
-      } else if (remoteInterpreterProcess.isRunning()) {
-        RemoteInterpreterService.Client client = null;
-        boolean broken = false;
-        try {
-          client = remoteInterpreterProcess.getClient();
-          if (client == null) {
-            // remote interpreter may not started yet or terminated.
-            continue;
-          }
-          List<String> resourceList = client.resourcePoolGetAll();
-          for (String res : resourceList) {
-            resourceSet.add(Resource.fromJson(res));
-          }
-        } catch (Exception e) {
-          logger.error(e.getMessage(), e);
-          broken = true;
-        } finally {
-          if (client != null) {
-            intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
-          }
-        }
-      }
-    }
-    return resourceSet;
-  }
-
-  public static void removeResourcesBelongsToNote(String noteId) {
-    removeResourcesBelongsToParagraph(noteId, null);
-  }
-
-  public static void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
-    for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
-      ResourceSet resourceSet = new ResourceSet();
-      RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
-      if (remoteInterpreterProcess == null) {
-        ResourcePool localPool = intpGroup.getResourcePool();
-        if (localPool != null) {
-          resourceSet.addAll(localPool.getAll());
-        }
-        if (noteId != null) {
-          resourceSet = resourceSet.filterByNoteId(noteId);
-        }
-        if (paragraphId != null) {
-          resourceSet = resourceSet.filterByParagraphId(paragraphId);
-        }
-
-        for (Resource r : resourceSet) {
-          localPool.remove(
-              r.getResourceId().getNoteId(),
-              r.getResourceId().getParagraphId(),
-              r.getResourceId().getName());
-        }
-      } else if (remoteInterpreterProcess.isRunning()) {
-        RemoteInterpreterService.Client client = null;
-        boolean broken = false;
-        try {
-          client = remoteInterpreterProcess.getClient();
-          List<String> resourceList = client.resourcePoolGetAll();
-          for (String res : resourceList) {
-            resourceSet.add(Resource.fromJson(res));
-          }
-
-          if (noteId != null) {
-            resourceSet = resourceSet.filterByNoteId(noteId);
-          }
-          if (paragraphId != null) {
-            resourceSet = resourceSet.filterByParagraphId(paragraphId);
-          }
-
-          for (Resource r : resourceSet) {
-            client.resourceRemove(
-                r.getResourceId().getNoteId(),
-                r.getResourceId().getParagraphId(),
-                r.getResourceId().getName());
-          }
-        } catch (Exception e) {
-          logger.error(e.getMessage(), e);
-          broken = true;
-        } finally {
-          if (client != null) {
-            intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
-          }
-        }
-      }
-    }
-  }
-}
-

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java

@@ -41,6 +41,7 @@ public abstract class Job {
   /**
    * Job status.
    *
+   * UNKNOWN - Job is not found in remote
    * READY - Job is not running, ready to run.
    * PENDING - Job is submitted to scheduler. but not running yet
    * RUNNING - Job is running.
@@ -48,8 +49,8 @@ public abstract class Job {
    * ERROR - Job finished run. with error
    * ABORT - Job finished by abort
    */
-  public static enum Status {
-    READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
+  public enum Status {
+    UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
 
     public boolean isReady() {
       return this == READY;
@@ -70,14 +71,14 @@ public abstract class Job {
   Date dateCreated;
   Date dateStarted;
   Date dateFinished;
-  Status status;
+  volatile Status status;
 
   static Logger LOGGER = LoggerFactory.getLogger(Job.class);
 
   transient boolean aborted = false;
 
-  private String errorMessage;
-  private transient Throwable exception;
+  private volatile String errorMessage;
+  private transient volatile Throwable exception;
   private transient JobListener listener;
   private long progressUpdateIntervalMs;
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java

@@ -17,24 +17,21 @@
 
 package org.apache.zeppelin.scheduler;
 
-import java.util.Collection;
 import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * TODO(moon) : add description.
+ * Factory class for creating schedulers
+ *
  */
 public class SchedulerFactory implements SchedulerListener {
   private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
-  ExecutorService executor;
-  Map<String, Scheduler> schedulers = new LinkedHashMap<>();
+  protected ExecutorService executor;
+  protected Map<String, Scheduler> schedulers = new LinkedHashMap<>();
 
   private static SchedulerFactory singleton;
   private static Long singletonLock = new Long(0);
@@ -54,17 +51,17 @@ public class SchedulerFactory implements SchedulerListener {
     return singleton;
   }
 
-  public SchedulerFactory() throws Exception {
-    executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100);
+  SchedulerFactory() throws Exception {
+    executor = ExecutorFactory.singleton().createOrGet("SchedulerFactory", 100);
   }
 
   public void destroy() {
-    ExecutorFactory.singleton().shutdown("schedulerFactory");
+    ExecutorFactory.singleton().shutdown("SchedulerFactory");
   }
 
   public Scheduler createOrGetFIFOScheduler(String name) {
     synchronized (schedulers) {
-      if (schedulers.containsKey(name) == false) {
+      if (!schedulers.containsKey(name)) {
         Scheduler s = new FIFOScheduler(name, executor, this);
         schedulers.put(name, s);
         executor.execute(s);
@@ -75,7 +72,7 @@ public class SchedulerFactory implements SchedulerListener {
 
   public Scheduler createOrGetParallelScheduler(String name, int maxConcurrency) {
     synchronized (schedulers) {
-      if (schedulers.containsKey(name) == false) {
+      if (!schedulers.containsKey(name)) {
         Scheduler s = new ParallelScheduler(name, executor, this, maxConcurrency);
         schedulers.put(name, s);
         executor.execute(s);
@@ -84,60 +81,38 @@ public class SchedulerFactory implements SchedulerListener {
     }
   }
 
-  public Scheduler createOrGetRemoteScheduler(
-      String name,
-      String noteId,
-      RemoteInterpreterProcess interpreterProcess,
-      int maxConcurrency) {
-
+  public Scheduler createOrGetScheduler(Scheduler scheduler) {
     synchronized (schedulers) {
-      if (schedulers.containsKey(name) == false) {
-        Scheduler s = new RemoteScheduler(
-            name,
-            executor,
-            noteId,
-            interpreterProcess,
-            this,
-            maxConcurrency);
-        schedulers.put(name, s);
-        executor.execute(s);
+      if (!schedulers.containsKey(scheduler.getName())) {
+        schedulers.put(scheduler.getName(), scheduler);
+        executor.execute(scheduler);
       }
-      return schedulers.get(name);
+      return schedulers.get(scheduler.getName());
     }
   }
 
-  public Scheduler removeScheduler(String name) {
+  public void removeScheduler(String name) {
     synchronized (schedulers) {
       Scheduler s = schedulers.remove(name);
       if (s != null) {
         s.stop();
       }
     }
-    return null;
   }
 
-  public Collection<Scheduler> listScheduler(String name) {
-    List<Scheduler> s = new LinkedList<>();
-    synchronized (schedulers) {
-      for (Scheduler ss : schedulers.values()) {
-        s.add(ss);
-      }
-    }
-    return s;
+  public ExecutorService getExecutor() {
+    return executor;
   }
 
   @Override
   public void jobStarted(Scheduler scheduler, Job job) {
-    logger.info("Job " + job.getJobName() + " started by scheduler " + scheduler.getName());
+    logger.info("Job " + job.getId() + " started by scheduler " + scheduler.getName());
 
   }
 
   @Override
   public void jobFinished(Scheduler scheduler, Job job) {
-    logger.info("Job " + job.getJobName() + " finished by scheduler " + scheduler.getName());
+    logger.info("Job " + job.getId() + " finished by scheduler " + scheduler.getName());
 
   }
-
-
-
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java

@@ -17,7 +17,6 @@
 package org.apache.zeppelin.tabledata;
 
 import org.apache.zeppelin.resource.Resource;
-import org.apache.zeppelin.resource.ResourcePoolUtils;
 
 import java.util.Iterator;
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.util;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Generate Tiny ID.
+ */
+public class IdHashes {
+  private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9',
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
+    'W', 'X', 'Y', 'Z'};
+
+  /**
+   * encodes the given string into the base of the dictionary provided in the constructor.
+   *
+   * @param value the number to encode.
+   * @return the encoded string.
+   */
+  private static String encode(Long value) {
+
+    List<Character> result = new ArrayList<>();
+    BigInteger base = new BigInteger("" + DICTIONARY.length);
+    int exponent = 1;
+    BigInteger remaining = new BigInteger(value.toString());
+    while (true) {
+      BigInteger a = base.pow(exponent); // 16^1 = 16
+      BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112
+      BigInteger c = base.pow(exponent - 1);
+      BigInteger d = b.divide(c);
+
+      // if d > dictionary.length, we have a problem. but BigInteger doesnt have
+      // a greater than method :-( hope for the best. theoretically, d is always
+      // an index of the dictionary!
+      result.add(DICTIONARY[d.intValue()]);
+      remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0
+
+      // finished?
+      if (remaining.equals(BigInteger.ZERO)) {
+        break;
+      }
+
+      exponent++;
+    }
+
+    // need to reverse it, since the start of the list contains the least significant values
+    StringBuffer sb = new StringBuffer();
+    for (int i = result.size() - 1; i >= 0; i--) {
+      sb.append(result.get(i));
+    }
+    return sb.toString();
+  }
+
+  public static String generateId() {
+    return encode(System.currentTimeMillis() + new Random().nextInt());
+  }
+}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java

@@ -17,7 +17,7 @@
 
 package org.apache.zeppelin.util;
 
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang.StringUtils;
 
 import java.io.IOException;
 import java.util.Properties;

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java

@@ -1,43 +0,0 @@
-package org.apache.zeppelin.interpreter;
-
-import java.util.Properties;
-
-/**
- *
- */
-public class DummyInterpreter extends Interpreter {
-
-  public DummyInterpreter(Properties property) {
-    super(property);
-  }
-
-  @Override
-  public void open() {
-
-  }
-
-  @Override
-  public void close() {
-
-  }
-
-  @Override
-  public InterpreterResult interpret(String st, InterpreterContext context) {
-    return null;
-  }
-
-  @Override
-  public void cancel(InterpreterContext context) {
-
-  }
-
-  @Override
-  public FormType getFormType() {
-    return null;
-  }
-
-  @Override
-  public int getProgress(InterpreterContext context) {
-    return 0;
-  }
-}

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java

@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.junit.After;
 import org.junit.Before;
@@ -29,7 +30,7 @@ import org.junit.Test;
 public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener {
   private File tmpDir;
   private File fileChanged;
-  private int numChanged;
+  private AtomicInteger numChanged;
   private InterpreterOutputChangeWatcher watcher;
 
   @Before
@@ -40,7 +41,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
     tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
     tmpDir.mkdirs();
     fileChanged = null;
-    numChanged = 0;
+    numChanged = new AtomicInteger(0);
   }
 
   @After
@@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
   @Test
   public void test() throws IOException, InterruptedException {
     assertNull(fileChanged);
-    assertEquals(0, numChanged);
+    assertEquals(0, numChanged.get());
 
     Thread.sleep(1000);
     // create new file
@@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
     }
 
     assertNotNull(fileChanged);
-    assertEquals(1, numChanged);
+    assertEquals(1, numChanged.get());
   }
 
 
   @Override
   public void fileChanged(File file) {
     fileChanged = file;
-    numChanged++;
+    numChanged.incrementAndGet();
 
     synchronized(this) {
       notify();

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java

@@ -24,6 +24,7 @@ import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 
+//TODO(zjffdu) add more test for Interpreter which is a very important class
 public class InterpreterTest {
 
   @Test
@@ -85,4 +86,41 @@ public class InterpreterTest {
     );
   }
 
+  public static class DummyInterpreter extends Interpreter {
+
+    public DummyInterpreter(Properties property) {
+      super(property);
+    }
+
+    @Override
+    public void open() {
+
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public InterpreterResult interpret(String st, InterpreterContext context) {
+      return null;
+    }
+
+    @Override
+    public void cancel(InterpreterContext context) {
+
+    }
+
+    @Override
+    public FormType getFormType() {
+      return null;
+    }
+
+    @Override
+    public int getProgress(InterpreterContext context) {
+      return 0;
+    }
+  }
+
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java

@@ -17,12 +17,11 @@
 
 package org.apache.zeppelin.interpreter.remote;
 
-import static org.junit.Assert.assertTrue;
+import org.junit.Test;
 
 import java.io.IOException;
 
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
-import org.junit.Test;
+import static org.junit.Assert.assertTrue;
 
 public class RemoteInterpreterUtilsTest {
 

zeppelin-interpreter/src/test/resources/conf/interpreter.json

@@ -0,0 +1,115 @@
+{
+  "interpreterSettings": {
+    "2C3RWCVAG": {
+      "id": "2C3RWCVAG",
+      "name": "test",
+      "group": "test",
+      "properties": {
+        "property_1": "value_1",
+        "property_2": "new_value_2",
+        "property_3": "value_3"
+      },
+      "status": "READY",
+      "interpreterGroup": [
+        {
+          "name": "echo",
+          "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+          "defaultInterpreter": true,
+          "editor": {
+            "language": "java",
+            "editOnDblClick": false
+          }
+        }
+      ],
+      "dependencies": [],
+      "option": {
+        "remote": true,
+        "port": -1,
+        "perNote": "shared",
+        "perUser": "shared",
+        "isExistingProcess": false,
+        "setPermission": false,
+        "users": [],
+        "isUserImpersonate": false
+      }
+    },
+
+    "2CKWE7B19": {
+      "id": "2CKWE7B19",
+      "name": "test2",
+      "group": "test",
+      "properties": {
+        "property_1": "value_1",
+        "property_2": "new_value_2",
+        "property_3": "value_3"
+      },
+      "status": "READY",
+      "interpreterGroup": [
+        {
+          "name": "echo",
+          "class": "org.apache.zeppelin.interpreter.EchoInterpreter",
+          "defaultInterpreter": true,
+          "editor": {
+            "language": "java",
+            "editOnDblClick": false
+          }
+        }
+      ],
+      "dependencies": [],
+      "option": {
+        "remote": true,
+        "port": -1,
+        "perNote": "shared",
+        "perUser": "shared",
+        "isExistingProcess": false,
+        "setPermission": false,
+        "users": [],
+        "isUserImpersonate": false
+      }
+    }
+  },
+  "interpreterBindings": {
+    "2C6793KRV": [
+      "2C48Y7FSJ",
+      "2C63XW4XE",
+      "2C66GE1VB",
+      "2C5VH924X",
+      "2C4BJDRRZ",
+      "2C3SQSB7V",
+      "2C4HKDCQW",
+      "2C3DR183X",
+      "2C66Z9XPQ",
+      "2C3PTPMUH",
+      "2C69WE69N",
+      "2C5SRRXHM",
+      "2C4ZD49PF",
+      "2C6V3D44K",
+      "2C4UB1UZA",
+      "2C5S1R21W",
+      "2C5DCRVGM",
+      "2C686X8ZH",
+      "2C3RWCVAG",
+      "2C3JKFMJU",
+      "2C3VECEG2"
+    ]
+  },
+  "interpreterRepositories": [
+    {
+      "id": "central",
+      "type": "default",
+      "url": "http://repo1.maven.org/maven2/",
+      "releasePolicy": {
+        "enabled": true,
+        "updatePolicy": "daily",
+        "checksumPolicy": "warn"
+      },
+      "snapshotPolicy": {
+        "enabled": true,
+        "updatePolicy": "daily",
+        "checksumPolicy": "warn"
+      },
+      "mirroredRepositories": [],
+      "repositoryManager": false
+    }
+  ]
+}

zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json

@@ -0,0 +1,42 @@
+[
+  {
+    "group": "test",
+    "name": "double_echo",
+    "className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
+    "properties": {
+      "property_1": {
+        "envName": "PROPERTY_1",
+        "propertyName": "property_1",
+        "defaultValue": "value_1",
+        "description": "desc_1"
+      },
+      "property_2": {
+        "envName": "PROPERTY_2",
+        "propertyName": "property_2",
+        "defaultValue": "value_2",
+        "description": "desc_2"
+      }
+    }
+  },
+
+  {
+    "group": "test",
+    "name": "echo",
+    "defaultInterpreter": true,
+    "className": "org.apache.zeppelin.interpreter.EchoInterpreter",
+    "properties": {
+      "property_1": {
+        "envName": "PROPERTY_1",
+        "propertyName": "property_1",
+        "defaultValue": "value_1",
+        "description": "desc_1"
+      },
+      "property_2": {
+        "envName": "PROPERTY_2",
+        "propertyName": "property_2",
+        "defaultValue": "value_2",
+        "description": "desc_2"
+      }
+    }
+  }
+]

zeppelin-interpreter/src/test/resources/log4j.properties

@@ -26,4 +26,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
 #
 
 # Root logger option
-log4j.rootLogger=INFO, stdout
+log4j.rootLogger=INFO, stdout
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG

zeppelin-jupyter/pom.xml

@@ -52,6 +52,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
+      <version>15.0</version>
     </dependency>
 
     <dependency>

zeppelin-server/pom.xml

@@ -87,7 +87,7 @@
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>zeppelin-zengine</artifactId>
-      <version>${project.version}</version>
+      <version>0.8.0-SNAPSHOT</version>
       <exclusions>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
@@ -195,101 +195,12 @@
       <artifactId>websocket-server</artifactId>
       <version>${jetty.version}</version>
     </dependency>
-<!--
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-jsp</artifactId>
-      <version>${jetty.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.eclipse.jetty.orbit</groupId>
-          <artifactId>javax.servlet.jsp</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.eclipse.jetty.orbit</groupId>
-          <artifactId>javax.servlet</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.eclipse.jetty.orbit</groupId>
-          <artifactId>org.apache.jasper.glassfish</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
 
-    <dependency>
-      <groupId>org.eclipse.jetty.orbit</groupId>
-      <artifactId>javax.servlet.jsp</artifactId>
-      <version>2.2.0.v201112011158</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.eclipse.jetty.orbit</groupId>
-          <artifactId>javax.servlet</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
--->
     <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-common.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>jersey-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>jersey-json</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>jersey-server</artifactId>
-        </exclusion>
-
-        <exclusion>
-          <groupId>javax.servlet</groupId>
-          <artifactId>servlet-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.avro</groupId>
-          <artifactId>avro</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.jackrabbit</groupId>
-          <artifactId>jackrabbit-webdav</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>commons-httpclient</groupId>
-          <artifactId>commons-httpclient</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.zookeeper</groupId>
-          <artifactId>zookeeper</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.eclipse.jgit</groupId>
-          <artifactId>org.eclipse.jgit</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.jcraft</groupId>
-          <artifactId>jsch</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.commons</groupId>
-          <artifactId>commons-compress</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
     <dependency>
       <groupId>org.quartz-scheduler</groupId>
       <artifactId>quartz</artifactId>
@@ -364,6 +275,10 @@
           <groupId>org.apache.commons</groupId>
           <artifactId>commons-lang3</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 

zeppelin-server/src/main/java/org/apache/zeppelin/rest/GetUserList.java

@@ -202,6 +202,7 @@ public class GetUserList {
    */
   public List<String> getUserList(JdbcRealm obj) {
     List<String> userlist = new ArrayList<>();
+    Connection con = null;
     PreparedStatement ps = null;
     ResultSet rs = null;
     DataSource dataSource = null;
@@ -239,7 +240,7 @@ public class GetUserList {
     }
 
     try {
-      Connection con = dataSource.getConnection();
+      con = dataSource.getConnection();
       ps = con.prepareStatement(userquery);
       ps.setString(1, username);
       ps.setString(2, tablename);
@@ -252,6 +253,7 @@ public class GetUserList {
     } finally {
       JdbcUtils.closeResultSet(rs);
       JdbcUtils.closeStatement(ps);
+      JdbcUtils.closeConnection(con);
     }
     return userlist;
   }

zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java

@@ -185,7 +185,7 @@ public class InterpreterRestApi {
 
       String noteId = request == null ? null : request.getNoteId();
       if (null == noteId) {
-        interpreterSettingManager.close(setting);
+        interpreterSettingManager.close(settingId);
       } else {
         interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
       }
@@ -208,7 +208,7 @@ public class InterpreterRestApi {
   @GET
   @ZeppelinApi
   public Response listInterpreter(String message) {
-    Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
+    Map<String, InterpreterSetting> m = interpreterSettingManager.getInterpreterSettingTemplates();
     return new JsonResponse<>(Status.OK, "", m).build();
   }
 

zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java

@@ -98,6 +98,7 @@ public class NotebookRestApi {
     permissionsMap.put("owners", notebookAuthorization.getOwners(noteId));
     permissionsMap.put("readers", notebookAuthorization.getReaders(noteId));
     permissionsMap.put("writers", notebookAuthorization.getWriters(noteId));
+    permissionsMap.put("runners", notebookAuthorization.getRunners(noteId));
     return new JsonResponse<>(Status.OK, "", permissionsMap).build();
   }
 
@@ -165,6 +166,18 @@ public class NotebookRestApi {
       throw new ForbiddenException(errorMsg);
     }
   }
+
+  /**
+   * Check if the current user can run the given note.
+   */
+  private void checkIfUserCanRun(String noteId, String errorMsg) {
+    Set<String> userAndRoles = Sets.newHashSet();
+    userAndRoles.add(SecurityUtils.getPrincipal());
+    userAndRoles.addAll(SecurityUtils.getRoles());
+    if (!notebookAuthorization.hasRunAuthorization(userAndRoles, noteId)) {
+      throw new ForbiddenException(errorMsg);
+    }
+  }
   
   private void checkIfNoteIsNotNull(Note note) {
     if (note == null) {
@@ -199,15 +212,31 @@ public class NotebookRestApi {
     HashMap<String, HashSet<String>> permMap =
         gson.fromJson(req, new TypeToken<HashMap<String, HashSet<String>>>() {}.getType());
     Note note = notebook.getNote(noteId);
-    
-    LOG.info("Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"),
-        permMap.get("readers"), permMap.get("writers"));
+
+    LOG.info("Set permissions {} {} {} {} {} {}", noteId, principal, permMap.get("owners"),
+            permMap.get("readers"), permMap.get("runners"), permMap.get("writers"));
 
     HashSet<String> readers = permMap.get("readers");
+    HashSet<String> runners = permMap.get("runners");
     HashSet<String> owners = permMap.get("owners");
     HashSet<String> writers = permMap.get("writers");
-    // Set readers, if writers and owners is empty -> set to user requesting the change
+    // Set readers, if runners, writers and owners is empty -> set to user requesting the change
     if (readers != null && !readers.isEmpty()) {
+      if (runners.isEmpty()) {
+        runners = Sets.newHashSet(SecurityUtils.getPrincipal());
+      }
+      if (writers.isEmpty()) {
+        writers = Sets.newHashSet(SecurityUtils.getPrincipal());
+      }
+      if (owners.isEmpty()) {
+        owners = Sets.newHashSet(SecurityUtils.getPrincipal());
+      }
+    }
+    // Set runners, if writers and owners is empty -> set to user requesting the change
+    if (runners != null && !runners.isEmpty()) {
+      if (writers.isEmpty()) {
+        writers = Sets.newHashSet(SecurityUtils.getPrincipal());
+      }
       if (owners.isEmpty()) {
         owners = Sets.newHashSet(SecurityUtils.getPrincipal());
       }
@@ -220,10 +249,12 @@ public class NotebookRestApi {
     }
 
     notebookAuthorization.setReaders(noteId, readers);
+    notebookAuthorization.setRunners(noteId, runners);
     notebookAuthorization.setWriters(noteId, writers);
     notebookAuthorization.setOwners(noteId, owners);
-    LOG.debug("After set permissions {} {} {}", notebookAuthorization.getOwners(noteId),
-        notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId));
+    LOG.debug("After set permissions {} {} {} {}", notebookAuthorization.getOwners(noteId),
+            notebookAuthorization.getReaders(noteId), notebookAuthorization.getRunners(noteId),
+            notebookAuthorization.getWriters(noteId));
     AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
     note.persist(subject);
     notebookServer.broadcastNote(note);
@@ -589,7 +620,7 @@ public class NotebookRestApi {
     Note note = notebook.getNote(noteId);
     AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note");
 
     try {
       note.runAll(subject);
@@ -616,7 +647,7 @@ public class NotebookRestApi {
     LOG.info("stop note jobs {} ", noteId);
     Note note = notebook.getNote(noteId);
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot stop this job for this note");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop this job for this note");
 
     for (Paragraph p : note.getParagraphs()) {
       if (!p.isTerminated()) {
@@ -690,7 +721,7 @@ public class NotebookRestApi {
 
     Note note = notebook.getNote(noteId);
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot run job for this note");
     Paragraph paragraph = note.getParagraph(paragraphId);
     checkIfParagraphIsNotNull(paragraph);
 
@@ -728,7 +759,7 @@ public class NotebookRestApi {
 
     Note note = notebook.getNote(noteId);
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run paragraph");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot run paragraph");
     Paragraph paragraph = note.getParagraph(paragraphId);
     checkIfParagraphIsNotNull(paragraph);
 
@@ -766,7 +797,7 @@ public class NotebookRestApi {
     LOG.info("stop paragraph job {} ", noteId);
     Note note = notebook.getNote(noteId);
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot stop paragraph");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot stop paragraph");
     Paragraph p = note.getParagraph(paragraphId);
     checkIfParagraphIsNotNull(p);
     p.abort();
@@ -791,7 +822,7 @@ public class NotebookRestApi {
 
     Note note = notebook.getNote(noteId);
     checkIfNoteIsNotNull(note);
-    checkIfUserCanWrite(noteId, "Insufficient privileges you cannot set a cron job for this note");
+    checkIfUserCanRun(noteId, "Insufficient privileges you cannot set a cron job for this note");
 
     if (!CronExpression.isValidExpression(request.getCronString())) {
       return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build();
@@ -922,7 +953,8 @@ public class NotebookRestApi {
       String noteId = Id[0];
       if (!notebookAuthorization.isOwner(noteId, userAndRoles) &&
           !notebookAuthorization.isReader(noteId, userAndRoles) &&
-          !notebookAuthorization.isWriter(noteId, userAndRoles)) {
+          !notebookAuthorization.isWriter(noteId, userAndRoles) &&
+              !notebookAuthorization.isRunner(noteId, userAndRoles)) {
         notesFound.remove(i);
         i--;
       }

zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java

@@ -73,19 +73,19 @@ public class CorsFilter implements Filter {
   }
 
   private void addCorsHeaders(HttpServletResponse response, String origin) {
-    response.addHeader("Access-Control-Allow-Origin", origin);
-    response.addHeader("Access-Control-Allow-Credentials", "true");
-    response.addHeader("Access-Control-Allow-Headers", "authorization,Content-Type");
-    response.addHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, HEAD, DELETE");
+    response.setHeader("Access-Control-Allow-Origin", origin);
+    response.setHeader("Access-Control-Allow-Credentials", "true");
+    response.setHeader("Access-Control-Allow-Headers", "authorization,Content-Type");
+    response.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, HEAD, DELETE");
     DateFormat fullDateFormatEN =
         DateFormat.getDateTimeInstance(DateFormat.FULL, DateFormat.FULL, new Locale("EN", "en"));
-    response.addHeader("Date", fullDateFormatEN.format(new Date()));
+    response.setHeader("Date", fullDateFormatEN.format(new Date()));
     ZeppelinConfiguration zeppelinConfiguration = ZeppelinConfiguration.create();
-    response.addHeader("X-FRAME-OPTIONS", zeppelinConfiguration.getXFrameOptions());
+    response.setHeader("X-FRAME-OPTIONS", zeppelinConfiguration.getXFrameOptions());
     if (zeppelinConfiguration.useSsl()) {
-      response.addHeader("Strict-Transport-Security", zeppelinConfiguration.getStrictTransport());
+      response.setHeader("Strict-Transport-Security", zeppelinConfiguration.getStrictTransport());
     }
-    response.addHeader("X-XSS-Protection", zeppelinConfiguration.getXxssProtection());
+    response.setHeader("X-XSS-Protection", zeppelinConfiguration.getXxssProtection());
   }
 
   @Override

zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java

@@ -31,12 +31,10 @@ import org.apache.shiro.web.env.EnvironmentLoaderListener;
 import org.apache.shiro.web.servlet.ShiroFilter;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.dep.DependencyResolver;
 import org.apache.zeppelin.helium.Helium;
 import org.apache.zeppelin.helium.HeliumApplicationFactory;
 import org.apache.zeppelin.helium.HeliumBundleFactory;
 import org.apache.zeppelin.interpreter.InterpreterFactory;
-import org.apache.zeppelin.interpreter.InterpreterOption;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterSettingManager;
 import org.apache.zeppelin.notebook.Notebook;
@@ -93,13 +91,11 @@ public class ZeppelinServer extends Application {
   private NotebookRepoSync notebookRepo;
   private NotebookAuthorization notebookAuthorization;
   private Credentials credentials;
-  private DependencyResolver depResolver;
 
   public ZeppelinServer() throws Exception {
     ZeppelinConfiguration conf = ZeppelinConfiguration.create();
 
-    this.depResolver = new DependencyResolver(
-        conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
+
 
     InterpreterOutput.limit = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
 
@@ -129,27 +125,10 @@ public class ZeppelinServer extends Application {
           new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
     }
 
-    ZeppelinServer.helium = new Helium(
-        conf.getHeliumConfPath(),
-        conf.getHeliumRegistry(),
-        new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
-            "helium-registry-cache"),
-        heliumBundleFactory,
-        heliumApplicationFactory);
-
-    // create bundle
-    try {
-      heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
-    } catch (Exception e) {
-      LOG.error(e.getMessage(), e);
-    }
-
-    this.schedulerFactory = new SchedulerFactory();
-    this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
-        new InterpreterOption(true));
-    this.replFactory = new InterpreterFactory(conf, notebookWsServer,
-        notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
-        interpreterSettingManager);
+    this.schedulerFactory = SchedulerFactory.singleton();
+    this.interpreterSettingManager = new InterpreterSettingManager(conf, notebookWsServer,
+        notebookWsServer, notebookWsServer);
+    this.replFactory = new InterpreterFactory(interpreterSettingManager);
     this.notebookRepo = new NotebookRepoSync(conf);
     this.noteSearchService = new LuceneSearch();
     this.notebookAuthorization = NotebookAuthorization.init(conf);
@@ -158,6 +137,22 @@ public class ZeppelinServer extends Application {
         notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
             noteSearchService, notebookAuthorization, credentials);
 
+    ZeppelinServer.helium = new Helium(
+        conf.getHeliumConfPath(),
+        conf.getHeliumRegistry(),
+        new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
+            "helium-registry-cache"),
+        heliumBundleFactory,
+        heliumApplicationFactory,
+        interpreterSettingManager);
+
+    // create bundle
+    try {
+      heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+    }
+
     // to update notebook from application event from remote process.
     heliumApplicationFactory.setNotebook(notebook);
     // to update fire websocket event on application event.
@@ -206,7 +201,7 @@ public class ZeppelinServer extends Application {
         LOG.info("Shutting down Zeppelin Server ... ");
         try {
           jettyWebServer.stop();
-          notebook.getInterpreterSettingManager().shutdown();
+          notebook.getInterpreterSettingManager().close();
           notebook.close();
           Thread.sleep(3000);
         } catch (Exception e) {

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener;
 import org.apache.zeppelin.display.Input;
 import org.apache.zeppelin.helium.ApplicationEventListener;
 import org.apache.zeppelin.helium.HeliumPackage;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -59,7 +54,6 @@ import org.apache.zeppelin.notebook.NotebookEventListener;
 import org.apache.zeppelin.notebook.NotebookImportDeserializer;
 import org.apache.zeppelin.notebook.Paragraph;
 import org.apache.zeppelin.notebook.ParagraphJobListener;
-import org.apache.zeppelin.notebook.ParagraphRuntimeInfo;
 import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
 import org.apache.zeppelin.notebook.socket.Message;
 import org.apache.zeppelin.notebook.socket.Message.OP;
@@ -86,8 +80,6 @@ import org.slf4j.LoggerFactory;
 import com.google.common.collect.Queues;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
 import com.google.gson.reflect.TypeToken;
 
 /**
@@ -463,7 +455,8 @@ public class NotebookServer extends WebSocketServlet
     Notebook notebook = notebook();
     List<Note> notes = notebook.getAllNotes();
     for (Note note : notes) {
-      List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+      List<String> ids = notebook.getInterpreterSettingManager()
+          .getInterpreterBinding(note.getId());
       for (String id : ids) {
         if (id.equals(interpreterGroupId)) {
           broadcast(note.getId(), m);
@@ -751,6 +744,25 @@ public class NotebookServer extends WebSocketServlet
     return true;
   }
 
+  /**
+   * @return false if user doesn't have runner permission for this paragraph
+   */
+  private boolean hasParagraphRunnerPermission(NotebookSocket conn,
+                                               Notebook notebook, String noteId,
+                                               HashSet<String> userAndRoles,
+                                               String principal, String op)
+      throws IOException {
+
+    NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
+    if (!notebookAuthorization.isRunner(noteId, userAndRoles)) {
+      permissionError(conn, op, principal, userAndRoles,
+              notebookAuthorization.getOwners(noteId));
+      return false;
+    }
+
+    return true;
+  }
+
   /**
    * @return false if user doesn't have writer permission for this paragraph
    */
@@ -1003,7 +1015,7 @@ public class NotebookServer extends WebSocketServlet
         List<String> interpreterSettingIds = new LinkedList<>();
         interpreterSettingIds.add(defaultInterpreterId);
         for (String interpreterSettingId : notebook.getInterpreterSettingManager().
-            getDefaultInterpreterSettingList()) {
+            getInterpreterSettingIds()) {
           if (!interpreterSettingId.equals(defaultInterpreterId)) {
             interpreterSettingIds.add(interpreterSettingId);
           }
@@ -1186,6 +1198,9 @@ public class NotebookServer extends WebSocketServlet
     Map<String, Object> params = (Map<String, Object>) fromMessage.get("params");
     Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");
     String noteId = getOpenNoteId(conn);
+    if (noteId == null) {
+      noteId = (String) fromMessage.get("noteId");
+    }
 
     if (!hasParagraphWriterPermission(conn, notebook, noteId,
         userAndRoles, fromMessage.principal, "write")) {
@@ -1360,12 +1375,13 @@ public class NotebookServer extends WebSocketServlet
       List<InterpreterSetting> settings =
           notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
       for (InterpreterSetting setting : settings) {
-        if (setting.getInterpreterGroup(user, note.getId()) == null) {
+        if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) {
           continue;
         }
-        if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
+        if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId())
+            .getId())) {
           AngularObjectRegistry angularObjectRegistry =
-              setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+              setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
 
           // first trying to get local registry
           ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@@ -1402,12 +1418,13 @@ public class NotebookServer extends WebSocketServlet
         List<InterpreterSetting> settings =
             notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
         for (InterpreterSetting setting : settings) {
-          if (setting.getInterpreterGroup(user, n.getId()) == null) {
+          if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) {
             continue;
           }
-          if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
+          if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId())
+              .getId())) {
             AngularObjectRegistry angularObjectRegistry =
-                setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
+                setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
             this.broadcastExcept(n.getId(),
                 new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
                     .put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
@@ -1618,7 +1635,7 @@ public class NotebookServer extends WebSocketServlet
 
     String noteId = getOpenNoteId(conn);
 
-    if (!hasParagraphWriterPermission(conn, notebook, noteId,
+    if (!hasParagraphRunnerPermission(conn, notebook, noteId,
         userAndRoles, fromMessage.principal, "write")) {
       return;
     }
@@ -1636,7 +1653,7 @@ public class NotebookServer extends WebSocketServlet
       return;
     }
 
-    if (!hasParagraphWriterPermission(conn, notebook, noteId,
+    if (!hasParagraphRunnerPermission(conn, notebook, noteId,
         userAndRoles, fromMessage.principal, "run all paragraphs")) {
       return;
     }
@@ -1729,7 +1746,7 @@ public class NotebookServer extends WebSocketServlet
 
     String noteId = getOpenNoteId(conn);
 
-    if (!hasParagraphWriterPermission(conn, notebook, noteId,
+    if (!hasParagraphRunnerPermission(conn, notebook, noteId,
         userAndRoles, fromMessage.principal, "write")) {
       return;
     }
@@ -2057,7 +2074,7 @@ public class NotebookServer extends WebSocketServlet
       Set<String> userAndRoles = Sets.newHashSet();
       userAndRoles.add(SecurityUtils.getPrincipal());
       userAndRoles.addAll(SecurityUtils.getRoles());
-      if (!notebookIns.getNotebookAuthorization().hasWriteAuthorization(userAndRoles, noteId)) {
+      if (!notebookIns.getNotebookAuthorization().hasRunAuthorization(userAndRoles, noteId)) {
         throw new ForbiddenException(String.format("can't execute note %s", noteId));
       }
 
@@ -2280,13 +2297,13 @@ public class NotebookServer extends WebSocketServlet
 
     for (InterpreterSetting intpSetting : settings) {
       AngularObjectRegistry registry =
-          intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
+          intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
       List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
       for (AngularObject object : objects) {
         conn.send(serializeMessage(
             new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
                 .put("interpreterGroupId",
-                    intpSetting.getInterpreterGroup(user, note.getId()).getId())
+                    intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getId())
                 .put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
       }
     }
@@ -2332,7 +2349,7 @@ public class NotebookServer extends WebSocketServlet
       }
 
       List<String> settingIds =
-          notebook.getInterpreterSettingManager().getInterpreters(note.getId());
+          notebook.getInterpreterSettingManager().getInterpreterBinding(note.getId());
       for (String id : settingIds) {
         if (interpreterGroupId.contains(id)) {
           broadcast(note.getId(),

zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java

@@ -40,6 +40,7 @@ abstract public class AbstractZeppelinIT {
   protected static WebDriver driver;
 
   protected final static Logger LOG = LoggerFactory.getLogger(AbstractZeppelinIT.class);
+  protected static final long MIN_IMPLICIT_WAIT = 5;
   protected static final long MAX_IMPLICIT_WAIT = 30;
   protected static final long MAX_BROWSER_TIMEOUT_SEC = 30;
   protected static final long MAX_PARAGRAPH_TIMEOUT_SEC = 120;

zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java

@@ -141,6 +141,7 @@ public class WebDriverManager {
       fail();
     }
 
+    driver.manage().window().maximize();
     return driver;
   }
 

zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java

@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.AbstractZeppelinIT;
@@ -38,6 +37,7 @@ import org.junit.Test;
 import org.junit.rules.ErrorCollector;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Keys;
+import org.openqa.selenium.TimeoutException;
 import org.openqa.selenium.WebElement;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,12 +63,14 @@ public class AuthenticationIT extends AbstractZeppelinIT {
       "securityManager.sessionManager = $sessionManager\n" +
       "securityManager.sessionManager.globalSessionTimeout = 86400000\n" +
       "shiro.loginUrl = /api/login\n" +
+      "anyofroles = org.apache.zeppelin.utils.AnyOfRolesAuthorizationFilter\n" +
       "[roles]\n" +
       "admin = *\n" +
       "hr = *\n" +
       "finance = *\n" +
       "[urls]\n" +
       "/api/version = anon\n" +
+      "/api/interpreter/** = authc, anyofroles[admin, finance]\n" +
       "/** = authc";
 
   static String originalShiro = "";
@@ -182,6 +184,62 @@ public class AuthenticationIT extends AbstractZeppelinIT {
     }
   }
 
+  @Test
+  public void testAnyOfRoles() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      AuthenticationIT authenticationIT = new AuthenticationIT();
+      authenticationIT.authenticationUser("admin", "password1");
+
+      pollingWait(By.xpath("//div/button[contains(@class, 'nav-btn dropdown-toggle ng-scope')]"),
+          MAX_BROWSER_TIMEOUT_SEC).click();
+      clickAndWait(By.xpath("//li/a[contains(@href, '#/interpreter')]"));
+
+      collector.checkThat("Check is user has permission to view this page", true,
+          CoreMatchers.equalTo(pollingWait(By.xpath(
+              "//div[@id='main']/div/div[2]"),
+              MIN_IMPLICIT_WAIT).isDisplayed())
+      );
+
+      authenticationIT.logoutUser("admin");
+
+      authenticationIT.authenticationUser("finance1", "finance1");
+
+      pollingWait(By.xpath("//div/button[contains(@class, 'nav-btn dropdown-toggle ng-scope')]"),
+          MAX_BROWSER_TIMEOUT_SEC).click();
+      clickAndWait(By.xpath("//li/a[contains(@href, '#/interpreter')]"));
+
+      collector.checkThat("Check is user has permission to view this page", true,
+          CoreMatchers.equalTo(pollingWait(By.xpath(
+              "//div[@id='main']/div/div[2]"),
+              MIN_IMPLICIT_WAIT).isDisplayed())
+      );
+      
+      authenticationIT.logoutUser("finance1");
+
+      authenticationIT.authenticationUser("hr1", "hr1");
+
+      pollingWait(By.xpath("//div/button[contains(@class, 'nav-btn dropdown-toggle ng-scope')]"),
+          MAX_BROWSER_TIMEOUT_SEC).click();
+      clickAndWait(By.xpath("//li/a[contains(@href, '#/interpreter')]"));
+
+      try {
+        collector.checkThat("Check is user has permission to view this page",
+            true, CoreMatchers.equalTo(
+                pollingWait(By.xpath("//li[contains(@class, 'ng-toast__message')]//span/span"),
+                    MIN_IMPLICIT_WAIT).isDisplayed()));
+      } catch (TimeoutException e) {
+        throw new Exception("Expected ngToast not found", e);
+      }
+      authenticationIT.logoutUser("hr1");
+
+    } catch (Exception e) {
+      handleException("Exception in AuthenticationIT while testAnyOfRoles ", e);
+    }
+  }
+
   @Test
   public void testGroupPermission() throws Exception {
     if (!endToEndTestEnabled()) {
@@ -200,6 +258,8 @@ public class AuthenticationIT extends AbstractZeppelinIT {
           MAX_BROWSER_TIMEOUT_SEC).sendKeys("finance ");
       pollingWait(By.xpath(".//*[@id='selectReaders']/following::span//input"),
           MAX_BROWSER_TIMEOUT_SEC).sendKeys("finance ");
+      pollingWait(By.xpath(".//*[@id='selectRunners']/following::span//input"),
+              MAX_BROWSER_TIMEOUT_SEC).sendKeys("finance ");
       pollingWait(By.xpath(".//*[@id='selectWriters']/following::span//input"),
           MAX_BROWSER_TIMEOUT_SEC).sendKeys("finance ");
       pollingWait(By.xpath("//button[@ng-click='savePermissions()']"), MAX_BROWSER_TIMEOUT_SEC)
@@ -254,7 +314,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
 
 
     } catch (Exception e) {
-      handleException("Exception in ParagraphActionsIT while testGroupPermission ", e);
+      handleException("Exception in AuthenticationIT while testGroupPermission ", e);
     }
   }
 

zeppelin-server/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java

@@ -72,7 +72,7 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
   static String interpreterOptionPath = "";
   static String originalInterpreterOption = "";
 
-  static String cmdPsPython = "ps aux | grep 'zeppelin_python-' | grep -v 'grep' | wc -l";
+  static String cmdPsPython = "ps aux | grep 'zeppelin_ipython' | grep -v 'grep' | wc -l";
   static String cmdPsInterpreter = "ps aux | grep 'zeppelin/interpreter/python/*' |" +
       " sed -E '/grep|local-repo/d' | wc -l";
 
@@ -145,19 +145,19 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
   }
 
   private void logoutUser(String userName) throws URISyntaxException {
-    pollingWait(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
-        userName + "')]"), MAX_BROWSER_TIMEOUT_SEC).click();
-    pollingWait(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
-        userName + "')]//a[@ng-click='navbar.logout()']"), MAX_BROWSER_TIMEOUT_SEC).click();
-
-    By locator = By.xpath("//*[@id='loginModal']//div[contains(@class, 'modal-header')]/button");
-    WebElement element = (new WebDriverWait(driver, MAX_BROWSER_TIMEOUT_SEC))
-        .until(ExpectedConditions.visibilityOfElementLocated(locator));
-    if (element.isDisplayed()) {
+    ZeppelinITUtils.sleep(500, false);
+    driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
+        userName + "')]")).click();
+    ZeppelinITUtils.sleep(500, false);
+    driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
+        userName + "')]//a[@ng-click='navbar.logout()']")).click();
+    ZeppelinITUtils.sleep(2000, false);
+    if (driver.findElement(By.xpath("//*[@id='loginModal']//div[contains(@class, 'modal-header')]/button"))
+        .isDisplayed()) {
       driver.findElement(By.xpath("//*[@id='loginModal']//div[contains(@class, 'modal-header')]/button")).click();
     }
     driver.get(new URI(driver.getCurrentUrl()).resolve("/#/").toString());
-    ZeppelinITUtils.sleep(1000, false);
+    ZeppelinITUtils.sleep(500, false);
   }
 
   private void setPythonParagraph(int num, String text) {
@@ -199,7 +199,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
           "//div[@class='modal-dialog']//div[@class='bootstrap-dialog-footer-buttons']//button[contains(., 'OK')]"));
       clickAndWait(By.xpath("//a[@class='navbar-brand navbar-title'][contains(@href, '#/')]"));
       interpreterModeActionsIT.logoutUser("admin");
-
       //step 2: (user1) login, create a new note, run two paragraph with 'python', check result, check process, logout
       //paragraph: Check if the result is 'user1' in the second paragraph
       //System: Check if the number of python interpreter process is '1'

zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java

@@ -138,7 +138,7 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
       WebElement paragraph1Result = driver.findElement(By.xpath(
           getParagraphXPath(1) + "//div[contains(@id,\"_text\")]"));
       collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
-          paragraph1Result.getText().toString(), CoreMatchers.equalTo("test loop 0\ntest loop 1\ntest loop 2")
+          paragraph1Result.getText().toString(), CoreMatchers.containsString("test loop 0\ntest loop 1\ntest loop 2")
       );
 
       // the last statement's evaluation result is printed

zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java

@@ -211,6 +211,7 @@ public abstract class AbstractTestRestApi {
         // set spark home for pyspark
         sparkProperties.put("spark.home",
             new InterpreterProperty("spark.home", getSparkHome(), InterpreterPropertyType.TEXTAREA.getValue()));
+        sparkProperties.put("zeppelin.spark.useIPython",  new InterpreterProperty("zeppelin.spark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue()));
 
         sparkIntpSetting.setProperties(sparkProperties);
         pySpark = true;
@@ -233,6 +234,8 @@ public abstract class AbstractTestRestApi {
               new InterpreterProperty("spark.home", sparkHome, InterpreterPropertyType.TEXTAREA.getValue()));
           sparkProperties.put("zeppelin.spark.useHiveContext",
               new InterpreterProperty("zeppelin.spark.useHiveContext", false, InterpreterPropertyType.CHECKBOX.getValue()));
+          sparkProperties.put("zeppelin.spark.useIPython",  new InterpreterProperty("zeppelin.spark.useIPython", "false", InterpreterPropertyType.TEXTAREA.getValue()));
+
           pySpark = true;
           sparkR = true;
         }
@@ -304,10 +307,9 @@ public abstract class AbstractTestRestApi {
   protected static void shutDown() throws Exception {
     if (!wasRunning) {
       // restart interpreter to stop all interpreter processes
-      List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
-          .getDefaultInterpreterSettingList();
-      for (String setting : settingList) {
-        ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
+      List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
+      for (InterpreterSetting setting : settingList) {
+        ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
       }
       if (shiroIni != null) {
         FileUtils.deleteQuietly(shiroIni);

zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java

@@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
 
     // then
     assertThat(get, isAllowed());
-    assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
+    assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettingTemplates().size(),
         body.entrySet().size());
     get.releaseConnection();
   }
@@ -110,7 +110,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
   @Test
   public void testSettingsCRUD() throws IOException {
     // when: call create setting API
-    String rawRequest = "{\"name\":\"md2\",\"group\":\"md\"," +
+    String rawRequest = "{\"name\":\"md3\",\"group\":\"md\"," +
         "\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
         "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
         "\"dependencies\":[]," +
@@ -367,7 +367,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
 
   @Test
   public void testGetMetadataInfo() throws IOException {
-    String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
+    String jsonRequest = "{\"name\":\"spark_new\",\"group\":\"spark\"," +
             "\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
             "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
             "\"dependencies\":[]," +