Merge branch 'master' into ZEPPELIN-2813_refactoring_2

.github/PULL_REQUEST_TEMPLATE

@@ -14,7 +14,9 @@ First time? Check out the contributing guide - https://zeppelin.apache.org/contr
 * Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
 
 ### How should this be tested?
-Outline the steps to test the PR here.
+* First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration
+* Strongly recommended: add automated unit tests for any new or changed behavior
+* Outline any manual steps to test the PR here.
 
 ### Screenshots (if appropriate)
 

.travis.yml

@@ -37,7 +37,7 @@ addons:
 env:
   global:
     # Interpreters does not required by zeppelin-server integration tests
-    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
+    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
 
 matrix:
   include:
@@ -53,7 +53,7 @@ matrix:
       sudo: false
       dist: trusty
       jdk: "oraclejdk8"
-      env: WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
+      env: PYTHON="2" WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
       addons:
         apt:
           sources:
@@ -66,56 +66,57 @@ matrix:
     # Several tests were excluded from this configuration due to the following issues:
     # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
     # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
-    - jdk: "oraclejdk8"
+    - sudo: required
+      jdk: "oraclejdk8"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
 
     # Test selenium with spark module for 1.6.3
     - jdk: "oraclejdk7"
       dist: precise
-      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+      env: PYTHON="2" TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
 
     # Test interpreter modules
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+      env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="install -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
 
     # Test spark module for 2.2.0 with scala 2.11, livy
     - jdk: "oraclejdk8"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
 
     # Test spark module for 2.1.0 with scala 2.11, livy
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
 
     # Test spark module for 2.0.2 with scala 2.11
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 1.6.3 with scala 2.10
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 1.6.3 with scala 2.11
     - jdk: "oraclejdk7"
       dist: precise
-      env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test python/pyspark with python 2, livy 0.2
     - sudo: required
       dist: precise
       jdk: "oraclejdk7"
-      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
     # Test python/pyspark with python 3, livy 0.3
     - sudo: required
       dist: precise
       jdk: "oraclejdk7"
-      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
   # check files included in commit range, clear bower_components if a bower.json file has changed.
@@ -141,8 +142,9 @@ install:
 before_script:
   - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
   - if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
-  - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
+  - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-$LIVY_VER-bin; fi
   - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
+  - export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
   - echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh
   - tail conf/zeppelin-env.sh
@@ -162,15 +164,13 @@ after_success:
 after_failure:
   - echo "Travis exited with ${TRAVIS_TEST_RESULT}"
   - find . -name rat.txt | xargs cat
+  - cat logs/*
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
   - cat zeppelin-web/npm-debug.log
   - cat spark-*/logs/*
   - cat livy/target/tmp/*/output.log
-  - ls -R livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
-  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
-  - cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
   - cat livy/target/tmp/livy-int-test/*/output.log
-  - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
-  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
-  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
+  - ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stdout
+  - cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stderr

README.md

@@ -1,7 +1,7 @@
 # Apache Zeppelin
 
-**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
-**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
+**Documentation:** [User Guide](https://zeppelin.apache.org/docs/latest/index.html)<br/>
+**Mailing Lists:** [User and Dev mailing list](https://zeppelin.apache.org/community.html)<br/>
 **Continuous Integration:** [![Build Status](https://travis-ci.org/apache/zeppelin.svg?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
 **Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
 **Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
@@ -15,15 +15,15 @@ Core feature:
    * Built-in Apache Spark support
 
 
-To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org)
+To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](https://zeppelin.apache.org)
 
 
 ## Getting Started
 
 ### Install binary package
-Please go to [install](http://zeppelin.apache.org/docs/snapshot/install/install.html) to install Apache Zeppelin from binary package.
+Please go to [install](https://zeppelin.apache.org/docs/latest/install/install.html) to install Apache Zeppelin from binary package.
 
 ### Build from source
-Please check [Build from source](http://zeppelin.apache.org/docs/snapshot/install/build.html) to build Zeppelin from source.
+Please check [Build from source](https://zeppelin.apache.org/docs/latest/install/build.html) to build Zeppelin from source.
 
 

alluxio/pom.xml

@@ -20,10 +20,10 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <artifactId>zeppelin</artifactId>
+        <artifactId>interpreter-parent</artifactId>
         <groupId>org.apache.zeppelin</groupId>
         <version>0.8.0-SNAPSHOT</version>
-        <relativePath>..</relativePath>
+        <relativePath>../interpreter-parent</relativePath>
     </parent>
 
     <groupId>org.apache.zeppelin</groupId>
@@ -34,6 +34,7 @@
 
     <properties>
         <alluxio.version>1.0.0</alluxio.version>
+        <interpreter.name>alluxio</interpreter.name>
     </properties>
 
     <dependencies>
@@ -47,6 +48,7 @@
         <dependency>
               <groupId>com.google.guava</groupId>
               <artifactId>guava</artifactId>
+              <version>15.0</version>
         </dependency>
 
         <dependency>
@@ -128,54 +130,12 @@
         <plugins>
             <plugin>
                 <artifactId>maven-enforcer-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>enforce</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
             </plugin>
-
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>copy-dependencies</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy-dependencies</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>copy-artifact</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                            <artifactItems>
-                                <artifactItem>
-                                    <groupId>${project.groupId}</groupId>
-                                    <artifactId>${project.artifactId}</artifactId>
-                                    <version>${project.version}</version>
-                                    <type>${project.packaging}</type>
-                                </artifactItem>
-                            </artifactItems>
-                        </configuration>
-                    </execution>
-                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
             </plugin>
         </plugins>
     </build>

angular/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -32,6 +32,10 @@
   <version>0.8.0-SNAPSHOT</version>
   <name>Zeppelin: Angular interpreter</name>
 
+  <properties>
+    <interpreter.name>angular</interpreter.name>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>${project.groupId}</groupId>
@@ -61,54 +65,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

beam/README.md

@@ -8,7 +8,7 @@ Current interpreter implementation supports the static repl. It compiles the cod
 You have to first build the Beam interpreter by enable the **beam** profile as follows:
 
 ```
-mvn clean package -Pbeam -DskipTests
+mvn clean package -Pbeam -DskipTests -Pscala-2.10
 ```
 
 ### Notice

beam/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
   
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
   
   <groupId>org.apache.zeppelin</groupId>
@@ -35,12 +35,13 @@
   <properties>
     <beam.hadoop.version>2.3.0</beam.hadoop.version>
     <beam.spark.version>1.6.2</beam.spark.version>
-    <beam.beam.version>0.2.0-incubating</beam.beam.version>
+    <beam.beam.version>2.0.0</beam.beam.version>
 
     <!-- library versions -->
     <netty.version>4.1.1.Final</netty.version>
     <servlet.api.version>3.1.0</servlet.api.version>
     <commons.exec.version>1.3</commons.exec.version>
+    <interpreter.name>beam</interpreter.name>
   </properties>
   
   <dependencies>
@@ -211,6 +212,14 @@
       <version>${beam.beam.version}</version>
       <type>jar</type>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-runners-flink_${scala.binary.version}</artifactId>
+      <version>${beam.beam.version}</version>
+      <exclusions>
+      </exclusions>
+    </dependency>
   
     <dependency>
       <groupId>${project.groupId}</groupId>
@@ -232,69 +241,18 @@
     </dependency>
   
   </dependencies>
+
   <build>
     <plugins>
-  
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-deploy-plugin</artifactId>
-        <configuration>
-          <skip>true</skip>
-        </configuration>
-      </plugin>
-  
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
       </plugin>
-  
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
       </plugin>
-   
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
+      </plugin>
     </plugins>
   </build>
 </project>

bigquery/pom.xml

@@ -21,9 +21,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -41,6 +42,7 @@
     <!-- library versions -->
     <bigquery.api.version>v2-rev265-1.21.0</bigquery.api.version>
     <gson.version>2.6</gson.version>
+    <interpreter.name>bigquery</interpreter.name>
   </properties>
 
   <dependencies>
@@ -99,12 +101,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
 
       <plugin>
@@ -116,63 +118,22 @@
           </excludes>
         </configuration>
       </plugin>
-
       <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <mainClass>
+                org.apache.zeppelin.bigquery.BigQueryInterpreter
+              </mainClass>
+            </manifest>
+          </archive>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
       </plugin>
-      <plugin>
-            <artifactId>maven-assembly-plugin</artifactId>
-            <configuration>
-                <archive>
-                    <manifest>
-                        <mainClass>
-				org.apache.zeppelin.bigquery.BigQueryInterpreter                            
-                        </mainClass>
-                    </manifest>
-                </archive>
-                <descriptorRefs>
-                    <descriptorRef>jar-with-dependencies</descriptorRef>
-                </descriptorRefs>
-            </configuration>
-        </plugin>
     </plugins>
   </build>
+
 </project>

bin/common.sh

@@ -122,7 +122,11 @@ JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
 export JAVA_OPTS
 
 JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
-JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
+if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
+    JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
+else
+    JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties"
+fi
 export JAVA_INTP_OPTS
 
 

bin/interpreter.cmd

@@ -27,6 +27,7 @@ if /I "%~1"=="-d" (
     set INTERPRETER_ID=%~n2
 )
 if /I "%~1"=="-p" set PORT=%~2
+if /I "%~1"=="-c" set CALLBACK_HOST=%~2
 if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2
 shift
 goto loop
@@ -127,11 +128,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
 if defined SPARK_SUBMIT (
     set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%'
 
-    "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" %PORT%
+    "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT%
 ) else (
     set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"
 
-    "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% %PORT%
+    "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT%
 )
 
 exit /b

bin/interpreter.sh

@@ -23,7 +23,7 @@ function usage() {
     echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
 }
 
-while getopts "hp:d:l:v:u:g:" o; do
+while getopts "hc:p:d:l:v:u:g:" o; do
     case ${o} in
         h)
             usage
@@ -32,8 +32,11 @@ while getopts "hp:d:l:v:u:g:" o; do
         d)
             INTERPRETER_DIR=${OPTARG}
             ;;
+        c)
+            CALLBACK_HOST=${OPTARG} # This will be used callback host
+            ;;
         p)
-            PORT=${OPTARG}
+            PORT=${OPTARG} # This will be used callback port
             ;;
         l)
             LOCAL_INTERPRETER_REPO=${OPTARG}
@@ -140,7 +143,13 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
       export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
     fi
     unset PYSPARKPATH
+    export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
+  fi
 
+  if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+    ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
+    export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
+  else
     # autodetect HADOOP_CONF_HOME by heuristic
     if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then
       if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then
@@ -149,13 +158,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
         export HADOOP_CONF_DIR="/etc/hadoop/conf"
       fi
     fi
-
-    if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
-      ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
-    fi
-
-    export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
   fi
+
 elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
   if [[ -n "${HBASE_CONF_DIR}" ]]; then
     ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}"
@@ -202,12 +206,12 @@ fi
 
 if [[ -n "${SPARK_SUBMIT}" ]]; then
     if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]];  then
-       INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}`
+       INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}`
     else
-       INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${PORT}`
+       INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}`
     fi
 else
-    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} `
+    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} `
 fi
 
 if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then

bin/zeppelin-daemon.sh

@@ -67,6 +67,10 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
   ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
 fi
 
+if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+  ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+fi
+
 # Add jdbc connector jar
 # ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar"
 

bin/zeppelin.sh

@@ -73,6 +73,10 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
 
 ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
 
+if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+  ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+fi
+
 if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
   echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}"
   $(mkdir -p "${ZEPPELIN_LOG_DIR}")

cassandra/pom.xml

@@ -20,10 +20,10 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
-        <artifactId>zeppelin</artifactId>
+        <artifactId>interpreter-parent</artifactId>
         <groupId>org.apache.zeppelin</groupId>
         <version>0.8.0-SNAPSHOT</version>
-        <relativePath>..</relativePath>
+        <relativePath>../interpreter-parent</relativePath>
     </parent>
 
     <groupId>org.apache.zeppelin</groupId>
@@ -49,6 +49,7 @@
         <plugin.scala.version>2.15.2</plugin.scala.version>
         <plugin.scalatest.version>1.0</plugin.scalatest.version>
         <plugin.scalate.version>1.7.1</plugin.scalate.version>
+        <interpreter.name>cassandra</interpreter.name>
     </properties>
 
     <dependencies>
@@ -241,55 +242,14 @@
 
             <plugin>
                 <artifactId>maven-enforcer-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>enforce</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
             </plugin>
-
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>copy-dependencies</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy-dependencies</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>copy-artifact</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                            <artifactItems>
-                                <artifactItem>
-                                    <groupId>${project.groupId}</groupId>
-                                    <artifactId>${project.artifactId}</artifactId>
-                                    <version>${project.version}</version>
-                                    <type>${project.packaging}</type>
-                                </artifactItem>
-                            </artifactItems>
-                        </configuration>
-                    </execution>
-                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
             </plugin>
         </plugins>
+
     </build>
 </project>

conf/log4j_yarn_cluster.properties

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger = INFO, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
+

conf/shiro.ini.template

@@ -62,6 +62,14 @@ sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
 #cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
 #securityManager.cacheManager = $cacheManager
 
+### Enables 'HttpOnly' flag in Zeppelin cookies
+cookie = org.apache.shiro.web.servlet.SimpleCookie
+cookie.name = JSESSIONID
+cookie.httpOnly = true
+### Uncomment the below line only when Zeppelin is running over HTTPS
+#cookie.secure = true
+sessionManager.sessionIdCookie = $cookie
+
 securityManager.sessionManager = $sessionManager
 # 86,400,000 milliseconds = 24 hour
 securityManager.sessionManager.globalSessionTimeout = 86400000

conf/zeppelin-site.xml.template

@@ -138,6 +138,16 @@
 </property>
 -->
 
+<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
+<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
+<!--
+<property>
+  <name>zeppelin.notebook.s3.signerOverride</name>
+  <value>S3SignerType</value>
+  <description>optional override to control which signature algorithm should be used to sign AWS requests</description>
+</property>
+-->
+
 <!-- If using Azure for storage use the following settings -->
 <!--
 <property>
@@ -173,6 +183,26 @@
 </property>
 -->
 
+<!-- Notebook storage layer using hadoop compatible file system
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
+  <description>Hadoop compatible file system notebook persistence layer implementation, such as local file system, hdfs, azure wasb, s3 and etc.</description>
+</property>
+
+<property>
+  <name>zeppelin.server.kerberos.keytab</name>
+  <value></value>
+  <description>keytab for accessing kerberized hdfs</description>
+</property>
+
+<property>
+  <name>zeppelin.server.kerberos.principal</name>
+  <value></value>
+  <description>principal for accessing kerberized hdfs</description>
+</property>
+-->
+
 <!-- For connecting your Zeppelin with ZeppelinHub -->
 <!--
 <property>
@@ -396,4 +426,24 @@
 </property>
 -->
 
+<!--
+<property>
+  <name>zeppelin.server.strict.transport</name>
+  <value>max-age=631138519</value>
+  <description>The HTTP Strict-Transport-Security response header is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Enable this when Zeppelin is running on HTTPS. Value is in Seconds, the default value is equivalent to 20 years.</description>
+</property>
+-->
+<!--
+<property>
+  <name>zeppelin.server.xxss.protection</name>
+  <value>1</value>
+  <description>The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. When value is set to 1 and a cross-site scripting attack is detected, the browser will sanitize the page (remove the unsafe parts).</description>
+</property>
+-->
+<!--
+<property>
+  <name>zeppelin.interpreter.callback.portRange</name>
+  <value>10000:10010</value>
+</property>
+-->
 </configuration>

docs/_includes/themes/zeppelin/_navigation.html

@@ -95,6 +95,7 @@
                 <li><a href="{{BASE_PATH}}/setup/security/shiro_authentication.html">Shiro Authentication</a></li>
                 <li><a href="{{BASE_PATH}}/setup/security/notebook_authorization.html">Notebook Authorization</a></li>
                 <li><a href="{{BASE_PATH}}/setup/security/datasource_authorization.html">Data Source Authorization</a></li>
+                <li><a href="{{BASE_PATH}}/setup/security/http_security_headers.html">HTTP Security Headers</a></li>
                 <li role="separator" class="divider"></li>
                 <li class="title"><span>Notebook Storage</span></li>
                 <li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li>
@@ -137,6 +138,7 @@
                 <li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
+                <li><a href="{{BASE_PATH}}/interpreter/neo4j.html">Neo4j</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/pig.html">Pig</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, HAWQ</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/r.html">R</a></li>

docs/development/writing_zeppelin_interpreter.md

@@ -40,7 +40,49 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
 ## Make your own Interpreter
 
 Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
-You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
+For your interpreter project, you need to make `interpreter-parent` as your parent project and use plugin `maven-enforcer-plugin`, `maven-dependency-plugin` and `maven-resources-plugin`. Here's one sample pom.xml 
+
+```
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>interpreter-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.8.0-SNAPSHOT</version>
+        <relativePath>../interpreter-parent</relativePath>
+    </parent>
+    
+    ...
+    
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.zeppelin</groupId>
+            <artifactId>zeppelin-interpreter</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+          
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+    
+</project>
+```
+
+You should include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` as your interpreter's dependency in `pom.xml`. Bes
+And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
 
 There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
 

docs/index.md

@@ -98,6 +98,7 @@ limitations under the License.
   * [Shiro Authentication](./setup/security/shiro_authentication.html)
   * [Notebook Authorization](./setup/security/notebook_authorization.html)
   * [Data Source Authorization](./setup/security/datasource_authorization.html)
+  * [HTTP Security Headers](./setup/security/http_security_headers.html)
 * Notebook Storage: a guide about saving notebooks to external storage
   * [Git Storage](./setup/storage/storage.html#notebook-storage-in-local-git-repository)
   * [S3 Storage](./setup/storage/storage.html#notebook-storage-in-s3)
@@ -123,11 +124,6 @@ limitations under the License.
   * [Useful Developer Tools](./development/contribution/useful_developer_tools.html)
   * [How to Contribute (code)](./development/contribution/how_to_contribute_code.html)
   * [How to Contribute (website)](./development/contribution/how_to_contribute_website.html)
-
-#### External Resources
-  * [Mailing List](https://zeppelin.apache.org/community.html)
-  * [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
-  * [Stackoverflow Questions about Zeppelin (tag: `apache-zeppelin`)](http://stackoverflow.com/questions/tagged/apache-zeppelin)
   
 #### Available Interpreters 
   * [Alluxio](./interpreter/alluxio.html)
@@ -147,6 +143,7 @@ limitations under the License.
   * [Lens](./interpreter/lens.html)
   * [Livy](./interpreter/livy.html)
   * [markdown](./interpreter/markdown.html)
+  * [Neo4j](./interpreter/neo4j.html)
   * [Pig](./interpreter/pig.html)
   * [Postgresql, HAWQ](./interpreter/postgresql.html)
   * [Python](./interpreter/python.html)
@@ -156,3 +153,7 @@ limitations under the License.
   * [Shell](./interpreter/Shell.html)
   * [Spark](./interpreter/spark.html)
   
+#### External Resources
+  * [Mailing List](https://zeppelin.apache.org/community.html)
+  * [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
+  * [Stackoverflow Questions about Zeppelin (tag: `apache-zeppelin`)](http://stackoverflow.com/questions/tagged/apache-zeppelin)

docs/interpreter/beam.md

@@ -44,18 +44,10 @@ import java.io.Serializable;
 import java.util.Arrays;
 import java.util.List;
 import java.util.ArrayList;
-import org.apache.spark.api.java.*;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.SparkConf;
-import org.apache.spark.streaming.*;
-import org.apache.spark.SparkContext;
 import org.apache.beam.runners.direct.*;
 import org.apache.beam.sdk.runners.*;
 import org.apache.beam.sdk.options.*;
-import org.apache.beam.runners.spark.*;
-import org.apache.beam.runners.spark.io.ConsoleIO;
 import org.apache.beam.runners.flink.*;
-import org.apache.beam.runners.flink.examples.WordCount.Options;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -89,12 +81,12 @@ public class MinimalWordCount {
 	};  
   static final List<String> SENTENCES = Arrays.asList(SENTENCES_ARRAY);
   public static void main(String[] args) {
-    Options options = PipelineOptionsFactory.create().as(Options.class);
+    PipelineOptions options = PipelineOptionsFactory.create().as(PipelineOptions.class);
     options.setRunner(FlinkRunner.class);
     Pipeline p = Pipeline.create(options);
     p.apply(Create.of(SENTENCES).withCoder(StringUtf8Coder.of()))
          .apply("ExtractWords", ParDo.of(new DoFn<String, String>() {
-           @Override
+           @ProcessElement
            public void processElement(ProcessContext c) {
              for (String word : c.element().split("[^a-zA-Z']+")) {
                if (!word.isEmpty()) {
@@ -105,7 +97,7 @@ public class MinimalWordCount {
          }))
         .apply(Count.<String> perElement())
         .apply("FormatResults", ParDo.of(new DoFn<KV<String, Long>, String>() {
-          @Override
+          @ProcessElement
           public void processElement(DoFn<KV<String, Long>, String>.ProcessContext arg0)
             throws Exception {
             s.add("\n" + arg0.element().getKey() + "\t" + arg0.element().getValue());

docs/interpreter/cassandra.md

@@ -788,6 +788,29 @@ Below are the configuration parameters and their default value.
      </td>
      <td>DEFAULT</td>
    </tr>
+   <tr>
+     <td>cassandra.ssl.enabled</td>
+     <td>
+        Enable support for connecting to the Cassandra configured with SSL.
+        To connect to Cassandra configured with SSL use <strong>true</strong>
+        and provide a truststore file and password with following options.
+     </td>
+     <td>false</td>
+   </tr>
+   <tr>
+     <td>cassandra.ssl.truststore.path</td>
+     <td>
+        Filepath for the truststore file to use for connection to Cassandra with SSL.
+     </td>
+     <td></td>
+   </tr>
+   <tr>
+     <td>cassandra.ssl.truststore.password</td>
+     <td>
+        Password for the truststore file to use for connection to Cassandra with SSL.
+     </td>
+     <td></td>
+   </tr>
  </table>
 
 ## Change Log

docs/interpreter/livy.md

@@ -144,7 +144,12 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
     <td>zeppelin.livy.ssl.trustStorePassword</td>
     <td></td>
     <td>password for trustStore file. Used when livy ssl is enabled</td>
-  </tr>  
+  </tr>
+  <tr>
+    <td>zeppelin.livy.http.headers</td>
+    <td>key_1: value_1; key_2: value_2</td>
+    <td>custom http headers when calling livy rest api. Each http header is separated by `;`, and each header is one key value pair where key value is separated by `:`</td>
+  </tr>
 </table>
 
 **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.**

docs/interpreter/neo4j.md

@@ -0,0 +1,117 @@
+---
+layout: page
+title: "Neo4j Interpreter for Apache Zeppelin"
+description: "Neo4j is a native graph database, designed to store and process graphs from bottom to top."
+group: interpreter
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Neo4j Interpreter for Apache Zeppelin
+
+<div id="toc"></div>
+
+## Overview
+[Neo4j](https://neo4j.com/product/) is a native graph database, designed to store and process graphs from bottom to top.
+
+
+![Neo4j - Interpreter - Video]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif)
+
+## Configuration
+<table class="table-configuration">
+  <tr>
+    <th>Property</th>
+    <th>Default</th>
+    <th>Description</th>
+  </tr>
+  <tr>
+    <td>neo4j.url</td>
+    <td>bolt://localhost:7687</td>
+    <td>The Neo4j's BOLT url.</td>
+  </tr>
+  <tr>
+    <td>neo4j.auth.type</td>
+    <td>BASIC</td>
+    <td>The Neo4j's authentication type (NONE, BASIC).</td>
+  </tr>
+  <tr>
+    <td>neo4j.auth.user</td>
+    <td>neo4j</td>
+    <td>The Neo4j user name.</td>
+  </tr>
+  <tr>
+    <td>neo4j.auth.password</td>
+    <td>neo4j</td>
+    <td>The Neo4j user password.</td>
+  </tr>
+  <tr>
+    <td>neo4j.max.concurrency</td>
+    <td>50</td>
+    <td>Max concurrency call from Zeppelin to Neo4j server.</td>
+  </tr>
+</table>
+
+<center>
+  ![Interpreter configuration]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-config.png)
+</center>
+
+
+## Enabling the Neo4j Interpreter
+In a notebook, to enable the **Neo4j** interpreter, click the **Gear** icon and select **Neo4j**.
+
+## Using the Neo4j Interpreter
+In a paragraph, use `%neo4j` to select the Neo4j interpreter and then input the Cypher commands.
+For list of Cypher commands please refer to the official [Cyper Refcard](http://neo4j.com/docs/cypher-refcard/current/)
+
+```bash
+%neo4j
+//Sample the TrumpWorld dataset
+WITH
+'https://docs.google.com/spreadsheets/u/1/d/1Z5Vo5pbvxKJ5XpfALZXvCzW26Cl4we3OaN73K9Ae5Ss/export?format=csv&gid=1996904412' AS url
+LOAD CSV WITH HEADERS FROM url AS row
+RETURN row.`Entity A`, row.`Entity A Type`, row.`Entity B`, row.`Entity B Type`, row.Connection, row.`Source(s)`
+LIMIT 10
+```
+
+The Neo4j interpreter leverages the [Network display system](../usage/display_system/basic.html#network) allowing to visualize the them directly from the paragraph.
+
+
+### Write your Cypher queries and navigate your graph
+
+This query:
+
+```bash
+%neo4j
+MATCH (vp:Person {name:"VLADIMIR PUTIN"}), (dt:Person {name:"DONALD J. TRUMP"})
+MATCH path = allShortestPaths( (vp)-[*]-(dt) )
+RETURN path
+```
+produces the following result_
+![Neo4j - Graph - Result]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-graph.png)
+
+### Apply Zeppelin Dynamic Forms
+You can leverage [Zeppelin Dynamic Form](../usage/dynamic_form/intro.html) inside your queries. This query:
+
+```bash
+%neo4j
+MATCH (o:Organization)-[r]-()
+RETURN o.name, count(*), collect(distinct type(r)) AS types
+ORDER BY count(*) DESC
+LIMIT ${Show top=10}
+```
+
+produces the following result:
+![Neo4j - Zeppelin - Dynamic Forms]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png)
+

docs/interpreter/python.md

@@ -232,6 +232,70 @@ SELECT * FROM rates WHERE age < 40
 Otherwise it can be referred to as `%python.sql`
 
 
+## IPython Support
+
+IPython is more powerful than the default python interpreter with extra functionality. You can use IPython with Python2 or Python3 which depends on which python you set `zeppelin.python`.
+
+   **Pre-requests**
+   
+    - Jupyter `pip install jupyter`
+    - grpcio `pip install grpcio`
+
+If you already install anaconda, then you just need to install `grpcio` as Jupyter is already included in anaconda.
+
+In addition to all basic functions of the python interpreter, you can use all the IPython advanced features as you use it in Jupyter Notebook.
+
+e.g. 
+
+Use IPython magic
+
+```
+%python.ipython
+
+#python help
+range?
+
+#timeit
+%timeit range(100)
+```
+
+Use matplotlib 
+
+```
+%python.ipython
+
+
+%matplotlib inline
+import matplotlib.pyplot as plt
+
+print("hello world")
+data=[1,2,3,4]
+plt.figure()
+plt.plot(data)
+```
+
+We also make `ZeppelinContext` available in IPython Interpreter. You can use `ZeppelinContext` to create dynamic forms and display pandas DataFrame.
+
+e.g.
+
+Create dynamic form
+
+```
+z.input(name='my_name', defaultValue='hello')
+```
+
+Show pandas dataframe
+
+```
+import pandas as pd
+df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})
+z.show(df)
+
+```
+
+By default, we would use IPython in `%python.python` if IPython is available. Otherwise it would fall back to the original Python implementation.
+If you don't want to use IPython, then you can set `zeppelin.python.useIPython` as `false` in interpreter setting.
+
 ## Technical description
 
 For in-depth technical details on current implementation please refer to [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md).

docs/interpreter/shell.md

@@ -43,6 +43,11 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
     <td>60000</td>
     <td>Shell command time out in millisecs</td>
   </tr>
+  <tr>
+    <td>shell.working.directory.user.home</td>
+    <td>false</td>
+    <td>If this set to true, the shell's working directory will be set to user home</td>
+  </tr>
   <tr>
     <td>zeppelin.shell.auth.type</td>
     <td></td>

docs/interpreter/spark.md

@@ -181,6 +181,7 @@ For example,
  * **local[*]** in local mode
  * **spark://master:7077** in standalone cluster
  * **yarn-client** in Yarn client mode
+ * **yarn-cluster** in Yarn cluster mode
  * **mesos://host:5050** in Mesos cluster
 
 That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. 
@@ -188,6 +189,11 @@ For the further information about Spark & Zeppelin version compatibility, please
 
 > Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
 
+### 3. Yarn mode
+Zeppelin support both yarn client and yarn cluster mode (yarn cluster mode is supported from 0.8.0). For yarn mode, you must specify `SPARK_HOME` & `HADOOP_CONF_DIR`.
+You can either specify them in `zeppelin-env.sh`, or in interpreter setting page. Specifying them in `zeppelin-env.sh` means you can use only one version of `spark` & `hadoop`. Specifying them
+in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance.
+
 ## SparkContext, SQLContext, SparkSession, ZeppelinContext
 SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.
 Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x.
@@ -196,6 +202,13 @@ Staring from 0.6.1 SparkSession is available as variable `spark` when you are us
 
 <a name="dependencyloading"> </a>
 
+### How to pass property to SparkConf
+
+There're 2 kinds of properties that would be passed to SparkConf
+
+ * Standard spark property (prefix with `spark.`). e.g. `spark.executor.memory` will be passed to `SparkConf`
+ * Non-standard spark property (prefix with `zeppelin.spark.`).  e.g. `zeppelin.spark.property_1`, `property_1` will be passed to `SparkConf`
+
 ## Dependency Management
 There are two ways to load external libraries in Spark interpreter. First is using interpreter setting menu and second is loading Spark properties.
 
@@ -414,6 +427,12 @@ You can choose one of `shared`, `scoped` and `isolated` options wheh you configu
 Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). 
 It creates separated SparkContext per each notebook in `isolated` mode.
 
+## IPython support
+
+By default, zeppelin would use IPython in `pyspark` when IPython is available, Otherwise it would fall back to the original PySpark implementation.
+If you don't want to use IPython, then you can set `zeppelin.pyspark.useIPython` as `false` in interpreter setting. For the IPython features, you can refer doc
+[Python Interpreter](python.html)
+
 
 ## Setting up Zeppelin with Kerberos
 Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark on YARN:

docs/setup/basics/how_to_build.md

@@ -2,7 +2,7 @@
 layout: page
 title: "How to Build Zeppelin from source"
 description: "How to build Zeppelin from source"
-group: setup/basics 
+group: setup/basics
 ---
 <!--
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -207,8 +207,39 @@ Scalding Interpreter
 mvn clean package -Pscalding -DskipTests
 ```
 
+### Optional configurations
+
+Here are additional configurations that could be optionally tuned using the trailing `-D` option for maven commands
 
 
+Spark package
+
+```bash
+spark.archive # default spark-${spark.version}
+spark.src.download.url # default http://d3kbcqa49mib13.cloudfront.net/${spark.archive}.tgz
+spark.bin.download.url # default http://d3kbcqa49mib13.cloudfront.net/${spark.archive}-bin-without-hadoop.tgz
+```
+
+Py4J package
+
+```bash
+python.py4j.version # default 0.9.2
+pypi.repo.url # default https://pypi.python.org/packages
+python.py4j.repo.folder # default /64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3
+```
+
+final URL location for Py4J package will be produced as following:
+
+`${pypi.repo.url}${python.py4j.repo.folder}py4j-${python.py4j.version}.zip`
+
+
+Frontend Maven Plugin configurations
+
+```
+plugin.frontend.nodeDownloadRoot # default https://nodejs.org/dist/
+plugin.frontend.npmDownloadRoot # default http://registry.npmjs.org/npm/-/
+plugin.frontend.yarnDownloadRoot # default https://github.com/yarnpkg/yarn/releases/download/
+```
 
 ## Build requirements
 

docs/setup/operation/configuration.md

@@ -77,7 +77,19 @@ If both are defined, then the **environment variables** will take priority.
     <td>*</td>
     <td>Enables a way to specify a ',' separated list of allowed origins for REST and websockets. <br /> e.g. http://localhost:8080</td>
   </tr>
-    <tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_CREDENTIALS_PERSIST</h6></td>
+    <td><h6 class="properties">zeppelin.credentials.persist</h6></td>
+    <td>true</td>
+    <td>Persist credentials on a JSON file (credentials.json)</td>
+  </tr>  
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_CREDENTIALS_ENCRYPT_KEY</h6></td>
+    <td><h6 class="properties">zeppelin.credentials.encryptKey</h6></td>
+    <td></td>
+    <td>If provided, encrypt passwords on the credentials.json file (passwords will be stored as plain-text otherwise</td>
+  </tr>  
+  <tr>
     <td>N/A</td>
     <td><h6 class="properties">zeppelin.anonymous.allowed</h6></td>
     <td>true</td>
@@ -203,6 +215,12 @@ If both are defined, then the **environment variables** will take priority.
     <td>false</td>
     <td>Save notebooks to S3 with server-side encryption enabled</td>
   </tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
+    <td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
+    <td></td>
+    <td>Optional override to control which signature algorithm should be used to sign AWS requests</td>
+  </tr>
   <tr>
     <td><h6 class="properties">ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</h6></td>
     <td><h6 class="properties">zeppelin.notebook.azure.connectionString</h6></td>
@@ -411,6 +429,20 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order
 </property>
 ```
 
+### Storing user credentials
+
+In order to avoid having to re-enter credentials every time you restart/redeploy Zeppelin, you can store the user credentials. Zeppelin supports this via the ZEPPELIN_CREDENTIALS_PERSIST configuration.
+
+Please notice that passwords will be stored in *plain text* by default. To encrypt the passwords, use the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY config variable. This will encrypt passwords using the AES-128 algorithm.
+
+You can generate an appropriate encryption key any way you'd like - for instance, by using the openssl tool:
+
+```
+openssl enc -aes-128-cbc -k secret -P -md sha1
+```
+
+*Important*: storing your encryption key in a configuration file is _not advised_. Depending on your environment security needs, you may want to consider utilizing a credentials server, storing the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY as an OS env variable, or any other approach that would not colocate the encryption key and the encrypted content (the credentials.json file).
+
 
 ### Obfuscating Passwords using the Jetty Password Tool
 

docs/setup/security/http_security_headers.md

@@ -0,0 +1,110 @@
+---
+layout: page
+title: "Setting up HTTP Response Headers"
+description: "There are multiple HTTP Security Headers which can be configured in Apache Zeppelin. This page describes how to enable them by providing appropriate value in Zeppelin configuration file."
+group: setup/security
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+{% include JB/setup %}
+
+# Setting up HTTP Response Headers for Zeppelin 
+
+<div id="toc"></div>
+
+Apache Zeppelin can be configured to include HTTP Headers which aids in preventing Cross Site Scripting (XSS), Cross-Frame Scripting (XFS) and also enforces HTTP Strict Transport Security. Apache Zeppelin also has configuration available to set the Application Server Version to desired value.
+
+## Setting up HTTP Strict Transport Security (HSTS) Response Header
+
+Enabling HSTS Response Header prevents Man-in-the-middle attacks by automatically redirecting HTTP requests to HTTPS when Zeppelin Server is running on SSL. Read on how to configure SSL for Zeppelin [here] (../operation/configuration.html). Even if web page contains any resource which gets served over HTTP or any HTTP links, it will automatically be redirected to HTTPS for the target domain. 
+It also prevents MITM attack by not allowing User to override the invalid certificate message, when Attacker presents invalid SSL certificate to the User.  
+
+The following property needs to be updated in the zeppelin-site.xml in order to enable HSTS. You can choose appropriate value for "max-age".
+
+```
+<property>
+  <name>zeppelin.server.strict.transport</name>
+  <value>max-age=631138519</value>
+  <description>The HTTP Strict-Transport-Security response header is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Enable this when Zeppelin is running on HTTPS. Value is in Seconds, the default value is equivalent to 20 years.</description>
+</property>
+```
+
+
+Possible values are:
+
+* max-age=\<expire-time>
+* max-age=\<expire-time>; includeSubDomains
+* max-age=\<expire-time>; preload
+
+Read more about HSTS [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security).
+
+## Setting up X-XSS-PROTECTION Header
+
+The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari Web browsers that initiates configured action when they detect reflected cross-site scripting (XSS) attacks.
+ 
+The following property needs to be updated in the zeppelin-site.xml in order to set X-XSS-PROTECTION header. 
+
+```
+<property>
+  <name>zeppelin.server.xxss.protection</name>
+  <value>1; mode=block</value>
+  <description>The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. When value is set to 1 and a cross-site scripting attack is detected, the browser will sanitize the page (remove the unsafe parts).</description>
+</property>
+```
+
+
+You can choose appropriate value from below.
+
+* 0  (Disables XSS filtering)
+* 1  (Enables XSS filtering. If a cross-site scripting attack is detected, the browser will sanitize the page.)
+* 1; mode=block  (Enables XSS filtering. The browser will prevent rendering of the page if an attack is detected.)
+
+Read more about HTTP X-XSS-Protection response header [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection).
+
+## Setting up X-Frame-Options Header
+
+The X-Frame-Options HTTP response header can indicate browser to avoid clickjacking attacks, by ensuring that their content is not embedded into other sites in a `<frame>`,`<iframe>` or `<object>`.
+
+The following property needs to be updated in the zeppelin-site.xml in order to set X-Frame-Options header.
+
+```
+<property>
+  <name>zeppelin.server.xframe.options</name>
+  <value>SAMEORIGIN</value>
+  <description>The X-Frame-Options HTTP response header can be used to indicate whether or not a browser should be allowed to render a page in a frame/iframe/object.</description>
+</property>
+```
+
+
+You can choose appropriate value from below.
+
+* DENY
+* SAMEORIGIN
+* ALLOW-FROM _uri_
+
+## Setting up Server Header
+
+Security conscious organisations does not want to reveal the Application Server name and version to prevent finding this information easily by Attacker while fingerprinting the Application. The exact version number can tell an Attacker if the current Application Server is patched for or vulnerable to certain publicly known CVE associated to it.
+
+The following property needs to be updated in the zeppelin-site.xml in order to set Server header.
+
+```
+<property>
+    <name>zeppelin.server.jetty.name</name>
+    <value>Jetty(7.6.0.v20120127)</value>
+    <description>Hardcoding Application Server name to Prevent Fingerprinting</description>
+</property>
+```
+
+The value can be any "String".

docs/setup/security/notebook_authorization.md

@@ -36,13 +36,17 @@ As you can see, each Zeppelin notebooks has 3 entities :
 * Owners ( users or groups )
 * Readers ( users or groups )
 * Writers ( users or groups )
+* Runners ( users or groups )
 
 <center><img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/permission_setting.png"></center>
 
 Fill out the each forms with comma seperated **users** and **groups** configured in `conf/shiro.ini` file.
 If the form is empty (*), it means that any users can perform that operation.
 
-If someone who doesn't have **read** permission is trying to access the notebook or someone who doesn't have **write** permission is trying to edit the notebook, Zeppelin will ask to login or block the user.
+If someone who doesn't have **read** permission is trying to access the notebook or someone who doesn't have **write** permission is trying to edit the notebook,
+or someone who doesn't have **run** permission is trying to run a paragraph Zeppelin will ask to login or block the user.
+
+By default, owners and writers have **write** permission, owners, writers and runners have **run** permission, owners, writers, runners and readers have **read** permission
 
 <center><img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/insufficient_privileges.png"></center>
 
@@ -63,13 +67,13 @@ or set `zeppelin.notebook.public` property to `false` in `conf/zeppelin-site.xml
 </property>
 ```
 
-Behind the scenes, when you create a new note only the `owners` field is filled with current user, leaving `readers` and `writers` fields empty. All the notes with at least one empty authorization field are considered to be in `public` workspace. Thus when setting `zeppelin.notebook.public` (or corresponding `ZEPPELIN_NOTEBOOK_PUBLIC`) to false, newly created notes have `readers` and `writers` fields filled with current user, making note appear as in `private` workspace.
+Behind the scenes, when you create a new note only the `owners` field is filled with current user, leaving `readers`, `runners` and `writers` fields empty. All the notes with at least one empty authorization field are considered to be in `public` workspace. Thus when setting `zeppelin.notebook.public` (or corresponding `ZEPPELIN_NOTEBOOK_PUBLIC`) to false, newly created notes have `readers`, `runners`, `writers` fields filled with current user, making note appear as in `private` workspace.
 
 ## How it works
 In this section, we will explain the detail about how the notebook authorization works in backend side.
 
 ### NotebookServer
-The [NotebookServer](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java) classifies every notebook operations into three categories: **Read**, **Write**, **Manage**.
+The [NotebookServer](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java) classifies every notebook operations into three categories: **Read**, **Run**, **Write**, **Manage**.
 Before executing a notebook operation, it checks if the user and the groups associated with the `NotebookSocket` have permissions.
 For example, before executing a **Read** operation, it checks if the user and the groups have at least one entity that belongs to the **Reader** entities.
 

docs/setup/security/shiro_authentication.md

@@ -210,6 +210,21 @@ securityManager.realms = $zeppelinHubRealm
 
 > Note: ZeppelinHub is not releated to Apache Zeppelin project.
 
+## Secure Cookie for Zeppelin Sessions (optional)
+Zeppelin can be configured to set `HttpOnly` flag in the session cookie. With this configuration, Zeppelin cookies can 
+not be accessed via client side scripts thus preventing majority of Cross-site scripting (XSS) attacks.
+
+To enable secure cookie support via Shiro, add the following lines in `conf/shiro.ini` under `[main]` section, after
+defining a `sessionManager`.
+
+```
+cookie = org.apache.shiro.web.servlet.SimpleCookie
+cookie.name = JSESSIONID
+cookie.secure = true
+cookie.httpOnly = true
+sessionManager.sessionIdCookie = $cookie
+```
+
 ## Secure your Zeppelin information (optional)
 By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
 Sometimes you might want to hide these information for your use case.
@@ -226,7 +241,28 @@ Since Shiro provides **url-based security**, you can hide the information by com
 In this case, only who have `admin` role can see **Interpreter Setting**, **Credential** and **Configuration** information.
 If you want to grant this permission to other users, you can change **roles[ ]** as you defined at `[users]` section.
 
+### Apply multiple roles in Shiro configuration
+By default, Shiro will allow access to a URL if only user is part of "**all the roles**" defined like this:
+```
+[urls]
+
+/api/interpreter/** = authc, roles[admin, role1]
+```
+
+If there is a need that user with "**any of the defined roles**" should be allowed, then following Shiro configuration can be used:
+```
+[main]
+anyofroles = org.apache.zeppelin.utils.AnyOfRolesAuthorizationFilter
+
+[urls]
+
+/api/interpreter/** = authc, anyofroles[admin, role1]
+/api/configurations/** = authc, roles[admin]
+/api/credential/** = authc, roles[admin]
+```
+
 <br/>
+
 > **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file.
 
 

docs/setup/storage/storage.md

@@ -30,6 +30,7 @@ There are few notebook storage systems available for a use out of the box:
 
   * (default) use local file system and version it using local Git repository - `GitNotebookRepo`
   * all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
+  * all notes are saved in the notebook folder in hadoop compatible file system - `FileSystemNotebookRepo`
   * storage using Amazon S3 service - `S3NotebookRepo`
   * storage using Azure service - `AzureNotebookRepo`
   * storage using MongoDB - `MongoNotebookRepo`
@@ -51,6 +52,22 @@ To enable versioning for all your local notebooks though a standard Git reposito
 </property>
 ```
 
+</br>
+
+## Notebook Storage in hadoop compatible file system repository <a name="Hdfs"></a>
+
+Notes may be stored in hadoop compatible file system such as hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `FileSystemNotebookRepo`, then `zeppelin.notebook.dir` is the path on the hadoop compatible file system. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files.
+If your hadoop cluster is kerberized, then you need to specify `zeppelin.server.kerberos.keytab` and `zeppelin.server.kerberos.principal`
+
+```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
+  <description>hadoop compatible file system notebook persistence layer implementation</description>
+</property>
+```
+
+
 </br>
 
 ## Notebook Storage in S3 <a name="S3"></a>

docs/usage/rest_api/notebook.md

@@ -752,6 +752,59 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
     </tr>
   </table>
 
+<br/>
+
+### Update paragraph
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <td>Description</td>
+      <td>This ```PUT``` method update paragraph contents using given id, e.g. <code>{"text": "hello"}</code>
+      </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/paragraph/[paragraphId]```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>200</td>
+    </tr>
+    <tr>
+      <td>Bad Request code</td>
+      <td>400</td>
+    </tr>
+    <tr>
+      <td>Forbidden code</td>
+      <td>403</td>
+    </tr>
+    <tr>
+      <td>Not Found code</td>
+      <td>404</td>
+    </tr>
+    <tr>
+      <td>Fail code</td>
+      <td>500</td>
+    </tr>
+    <tr>
+      <td>sample JSON input</td>
+      <td><pre>
+{
+  "title": "Hello world",
+  "text": "println(\"hello world\")"
+}</pre></td>
+    </tr>
+    <tr>
+      <td>sample JSON response</td>
+      <td><pre>
+{
+  "status": "OK",
+  "message": ""
+  }
+}</pre></td>
+    </tr>
+  </table>
+
 <br/>
 ### Update paragraph configuration
   <table class="table-configuration">
@@ -1215,6 +1268,9 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
       "owners":[  
          "user1"
       ],
+      "runners":[
+         "user2"
+      ],
       "writers":[  
          "user2"
       ]
@@ -1259,6 +1315,9 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
   "owners": [
     "user2"
   ],
+  "runners":[
+    "user2"
+  ],
   "writers": [
     "user1"
   ]

elasticsearch/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <artifactId>zeppelin-elasticsearch</artifactId>
@@ -32,6 +32,7 @@
   <name>Zeppelin: Elasticsearch interpreter</name>
 
   <properties>
+    <interpreter.name>elasticsearch</interpreter.name>
     <elasticsearch.version>2.4.3</elasticsearch.version>
     <httpasyncclient.version>4.0.2</httpasyncclient.version>
     <guava.version>18.0</guava.version>
@@ -93,54 +94,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/elasticsearch</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/elasticsearch</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java

@@ -112,7 +112,7 @@ public class ElasticsearchInterpreter extends Interpreter {
 
   @Override
   public void open() {
-    logger.info("Properties: {}", getProperty());
+    logger.info("Properties: {}", getProperties());
 
     String clientType = getProperty(ELASTICSEARCH_CLIENT_TYPE);
     clientType = clientType == null ? null : clientType.toLowerCase();
@@ -123,15 +123,15 @@ public class ElasticsearchInterpreter extends Interpreter {
     catch (final NumberFormatException e) {
       this.resultSize = 10;
       logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " +
-          property.get(ELASTICSEARCH_RESULT_SIZE), e);
+          getProperty(ELASTICSEARCH_RESULT_SIZE), e);
     }
 
     try {
       if (StringUtils.isEmpty(clientType) || "transport".equals(clientType)) {
-        elsClient = new TransportBasedClient(getProperty());
+        elsClient = new TransportBasedClient(getProperties());
       }
       else if ("http".equals(clientType)) {
-        elsClient = new HttpBasedClient(getProperty());
+        elsClient = new HttpBasedClient(getProperties());
       }
       else {
         logger.error("Unknown type of Elasticsearch client: " + clientType);

file/pom.xml

@@ -20,9 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -33,11 +34,13 @@
 
   <properties>
     <!--library versions-->
+    <interpreter.name>file</interpreter.name>
     <ws.rsapi.version>2.0</ws.rsapi.version>
     <jersey.common.version>2.22.2</jersey.common.version>
 
     <!--plugin versions-->
     <plugin.surefire.version>2.18.1</plugin.surefire.version>
+    <interpreter.name>file</interpreter.name>
   </properties>
 
   <dependencies>
@@ -79,62 +82,14 @@
 
   <build>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <version>${plugin.surefire.version}</version>
-      </plugin>
-
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/file</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/file</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <!--<includeScope>runtime</includeScope>-->
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java

@@ -20,6 +20,7 @@ package org.apache.zeppelin.file;
 
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
@@ -86,7 +87,7 @@ public abstract class FileInterpreter extends Interpreter {
 
   // Functions that each file system implementation must override
 
-  public abstract String listAll(String path);
+  public abstract String listAll(String path) throws InterpreterException;
 
   public abstract boolean isDirectory(String path);
 

file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java

@@ -202,7 +202,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
     return "No such File or directory";
   }
 
-  public String listAll(String path) {
+  public String listAll(String path) throws InterpreterException {
     String all = "";
     if (exceptionOnConnect != null)
       return "Error connecting to provided endpoint.";

flink/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -35,6 +35,7 @@
 
   <properties>
     <!--library versions-->
+    <interpreter.name>flink</interpreter.name>
     <flink.version>1.1.3</flink.version>
     <flink.akka.version>2.3.7</flink.akka.version>
     <scala.macros.version>2.0.1</scala.macros.version>
@@ -280,68 +281,16 @@
 	</configuration>
       </plugin>
 
-      <plugin>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
-      </plugin>
+        <plugin>
+            <artifactId>maven-enforcer-plugin</artifactId>
+        </plugin>
+        <plugin>
+            <artifactId>maven-dependency-plugin</artifactId>
+        </plugin>
+        <plugin>
+            <artifactId>maven-resources-plugin</artifactId>
+        </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <configuration>
-          <forkCount>1</forkCount>
-          <reuseForks>false</reuseForks>
-          <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
-        </configuration>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>

flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java

@@ -17,7 +17,6 @@
  */
 package org.apache.zeppelin.flink;
 
-import java.lang.reflect.InvocationTargetException;
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -34,10 +33,8 @@ import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
-import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterUtils;
@@ -46,11 +43,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import scala.Console;
-import scala.None;
-import scala.Option;
 import scala.Some;
 import scala.collection.JavaConversions;
-import scala.collection.immutable.Nil;
 import scala.concurrent.duration.FiniteDuration;
 import scala.runtime.AbstractFunction0;
 import scala.tools.nsc.Settings;
@@ -80,7 +74,7 @@ public class FlinkInterpreter extends Interpreter {
   public void open() {
     out = new ByteArrayOutputStream();
     flinkConf = new org.apache.flink.configuration.Configuration();
-    Properties intpProperty = getProperty();
+    Properties intpProperty = getProperties();
     for (Object k : intpProperty.keySet()) {
       String key = (String) k;
       String val = toString(intpProperty.get(key));

geode/pom.xml

@@ -21,9 +21,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -34,6 +35,7 @@
 
   <properties>
     <!--library versions-->
+    <interpreter.name>geode</interpreter.name>
     <geode.version>1.1.0</geode.version>
     <commons.exec.version>1.3</commons.exec.version>
   </properties>
@@ -86,54 +88,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

groovy/pom.xml

@@ -20,11 +20,11 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <!--version>0.6.2</version-->
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -33,6 +33,10 @@
   <version>0.8.0-SNAPSHOT</version>
   <name>Zeppelin: Groovy interpreter</name>
 
+  <properties>
+    <interpreter.name>groovy</interpreter.name>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>${project.groupId}</groupId>
@@ -67,81 +71,14 @@
 
   <build>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.1</version>
-        <configuration>
-          <showDeprecation>true</showDeprecation>
-          <compilerArgs>
-            <!--arg>-verbose</arg-->
-            <arg>-Xlint:unchecked</arg>
-          </compilerArgs>
-        </configuration>
-      </plugin>
- 
-      <!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-checkstyle-plugin</artifactId>
-        <configuration>
-          <skip>true</skip>
-        </configuration>
-        <executions>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <version>1.3.1</version>            
-        <executions> 
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>2.8</version>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java

@@ -172,7 +172,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
   /**
    * starts or continues rendering html/angular and returns MarkupBuilder to build html.
    * <pre> g.html().with{
-   * 	h1("hello")
+   *  h1("hello")
    *  h2("world")
    * }</pre>
    */
@@ -316,12 +316,12 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
   @ZeppelinApi
   public void run(String noteId, String paragraphId, InterpreterContext context) {
     if (paragraphId.equals(context.getParagraphId())) {
-      throw new InterpreterException("Can not run current Paragraph");
+      throw new RuntimeException("Can not run current Paragraph");
     }
     List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, paragraphId,
         context);
     if (runners.size() <= 0) {
-      throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size());
+      throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size());
     }
     for (InterpreterContextRunner r : runners) {
       r.run();
@@ -338,7 +338,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
     List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
 
     if (runners.size() <= 0) {
-      throw new InterpreterException("Note " + noteId + " not found " + runners.size());
+      throw new RuntimeException("Note " + noteId + " not found " + runners.size());
     }
 
     for (InterpreterContextRunner r : runners) {

groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java

@@ -17,8 +17,6 @@
 
 package org.apache.zeppelin.groovy;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.io.StringWriter;
 import java.io.PrintWriter;
 import java.io.File;
@@ -26,10 +24,8 @@ import java.util.*;
 
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.scheduler.Scheduler;
@@ -40,7 +36,6 @@ import org.slf4j.LoggerFactory;
 import groovy.lang.GroovyShell;
 import groovy.lang.Script;
 import org.codehaus.groovy.control.CompilerConfiguration;
-import org.codehaus.groovy.runtime.ResourceGroovyMethods;
 import org.codehaus.groovy.runtime.StackTraceUtils;
 
 import java.util.concurrent.ConcurrentHashMap;
@@ -167,7 +162,7 @@ public class GroovyInterpreter extends Interpreter {
       //put shared bindings evaluated in this interpreter
       bindings.putAll(sharedBindings);
       //put predefined bindings
-      bindings.put("g", new GObject(log, out, this.getProperty(), contextInterpreter, bindings));
+      bindings.put("g", new GObject(log, out, this.getProperties(), contextInterpreter, bindings));
       bindings.put("out", new PrintWriter(out, true));
 
       script.run();
@@ -204,7 +199,7 @@ public class GroovyInterpreter extends Interpreter {
           Thread t = (Thread) object;
           t.dumpStack();
           t.interrupt();
-          //t.stop(); //TODO: need some way to terminate maybe through GObject..
+          //t.stop(); //TODO(dlukyanov): need some way to terminate maybe through GObject..
         } catch (Throwable t) {
           log.error("Failed to cancel script: " + t, t);
         }

hbase/pom.xml

@@ -20,9 +20,10 @@
      xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -33,6 +34,7 @@
 
   <properties>
     <!--library versions-->
+    <interpreter.name>hbase</interpreter.name>
     <hbase.hbase.version>1.0.0</hbase.hbase.version>
     <hbase.hadoop.version>2.6.0</hbase.hadoop.version>
     <jruby.version>1.6.8</jruby.version>
@@ -115,55 +117,14 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>
+
 </project>

hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java

@@ -68,7 +68,7 @@ public class HbaseInterpreter extends Interpreter {
   }
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
     this.scriptingContainer  = new ScriptingContainer(LocalContextScope.SINGLETON);
     this.writer = new StringWriter();
     scriptingContainer.setOutput(this.writer);
@@ -88,7 +88,7 @@ public class HbaseInterpreter extends Interpreter {
       }
 
       logger.info("Absolute Ruby Source:" + abs_ruby_src.toString());
-      // hirb.rb:41 requires the following system property to be set.
+      // hirb.rb:41 requires the following system properties to be set.
       Properties sysProps = System.getProperties();
       sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString());
 

hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java

@@ -15,6 +15,7 @@
 package org.apache.zeppelin.hbase;
 
 import org.apache.log4j.BasicConfigurator;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -35,7 +36,7 @@ public class HbaseInterpreterTest {
   private static HbaseInterpreter hbaseInterpreter;
 
   @BeforeClass
-  public static void setUp() throws NullPointerException {
+  public static void setUp() throws NullPointerException, InterpreterException {
     BasicConfigurator.configure();
     Properties properties = new Properties();
     properties.put("hbase.home", "");

helium-dev/pom.xml

@@ -23,8 +23,9 @@
 
   <parent>
     <groupId>org.apache.zeppelin</groupId>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -32,6 +33,10 @@
   <version>0.8.0-SNAPSHOT</version>
   <name>Zeppelin: Helium development interpreter</name>
 
+  <properties>
+    <interpreter.name>helium-dev</interpreter.name>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.zeppelin</groupId>
@@ -43,46 +48,14 @@
 
   <build>
     <plugins>
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+      </plugin>
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/helium-dev</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/helium-dev</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java

@@ -74,7 +74,8 @@ public class DevInterpreter extends Interpreter {
   }
 
   @Override
-  public InterpreterResult interpret(String st, InterpreterContext context) {
+  public InterpreterResult interpret(String st, InterpreterContext context)
+      throws InterpreterException {
     this.context = context;
     try {
       return interpreterEvent.interpret(st, context);

helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java

@@ -38,18 +38,18 @@ public class ZeppelinDevServer extends
 
   private DevInterpreter interpreter = null;
   private InterpreterOutput out;
-  public ZeppelinDevServer(int port) throws TException {
-    super(port);
+  public ZeppelinDevServer(int port) throws TException, IOException {
+    super(null, port);
   }
 
   @Override
-  protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+  protected Interpreter getInterpreter(String sessionId, String className) throws TException {
     synchronized (this) {
       InterpreterGroup interpreterGroup = getInterpreterGroup();
       if (interpreterGroup == null || interpreterGroup.isEmpty()) {
         createInterpreter(
             "dev",
-            sessionKey,
+            sessionId,
             DevInterpreter.class.getName(),
             new HashMap<String, String>(),
             "anonymous");
@@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
       }
     }
 
-    Interpreter intp = super.getInterpreter(sessionKey, className);
+    Interpreter intp = super.getInterpreter(sessionId, className);
     interpreter = (DevInterpreter) (
         ((LazyOpenInterpreter) intp).getInnerInterpreter());
     interpreter.setInterpreterEvent(this);
-    return super.getInterpreter(sessionKey, className);
+    return super.getInterpreter(sessionId, className);
   }
 
   @Override

ignite/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <artifactId>zeppelin-ignite_2.10</artifactId>
@@ -32,7 +32,8 @@
   <name>Zeppelin: Apache Ignite interpreter</name>
 
   <properties>
-    <ignite.version>1.9.0</ignite.version>
+    <interpreter.name>ignite</interpreter.name>
+    <ignite.version>2.2.0</ignite.version>
   </properties>
 
   <dependencies>
@@ -106,55 +107,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java

@@ -93,7 +93,7 @@ public class IgniteSqlInterpreter extends Interpreter {
   }
 
   @Override
-  public void close() {
+  public void close() throws InterpreterException {
     try {
       if (conn != null) {
         conn.close();

ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java

@@ -24,10 +24,10 @@ import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
@@ -83,7 +83,7 @@ public class IgniteSqlInterpreterTest {
   }
 
   @After
-  public void tearDown() {
+  public void tearDown() throws InterpreterException {
     intp.close();
     ignite.close();
   }

interpreter-parent/pom.xml

@@ -0,0 +1,146 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>zeppelin</artifactId>
+    <groupId>org.apache.zeppelin</groupId>
+    <version>0.8.0-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <groupId>org.apache.zeppelin</groupId>
+  <artifactId>interpreter-parent</artifactId>
+  <packaging>pom</packaging>
+  <version>0.8.0-SNAPSHOT</version>
+  <name>Zeppelin: Interpreter Parent</name>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>${project.groupId}</groupId>
+        <artifactId>zeppelin-interpreter</artifactId>
+        <version>0.8.0-SNAPSHOT</version>
+        <scope>provided</scope>
+      </dependency>
+
+      <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <version>${junit.version}</version>
+        <scope>test</scope>
+      </dependency>
+
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>${slf4j.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-log4j12</artifactId>
+        <version>${slf4j.version}</version>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <artifactId>maven-enforcer-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>enforce</id>
+              <phase>none</phase>
+            </execution>
+          </executions>
+        </plugin>
+
+        <plugin>
+          <artifactId>maven-dependency-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>copy-dependencies</id>
+              <phase>none</phase>
+              <configuration>
+                <skip>true</skip>
+              </configuration>
+            </execution>
+
+            <execution>
+              <id>copy-interpreter-dependencies</id>
+              <phase>package</phase>
+              <goals>
+                <goal>copy-dependencies</goal>
+              </goals>
+              <configuration>
+                <outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
+                <overWriteReleases>false</overWriteReleases>
+                <overWriteSnapshots>false</overWriteSnapshots>
+                <overWriteIfNewer>true</overWriteIfNewer>
+                <includeScope>runtime</includeScope>
+              </configuration>
+            </execution>
+            <execution>
+              <id>copy-artifact</id>
+              <phase>package</phase>
+              <goals>
+                <goal>copy</goal>
+              </goals>
+              <configuration>
+                <outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
+                <overWriteReleases>false</overWriteReleases>
+                <overWriteSnapshots>false</overWriteSnapshots>
+                <overWriteIfNewer>true</overWriteIfNewer>
+                <artifactItems>
+                  <artifactItem>
+                    <groupId>${project.groupId}</groupId>
+                    <artifactId>${project.artifactId}</artifactId>
+                    <version>${project.version}</version>
+                    <type>${project.packaging}</type>
+                  </artifactItem>
+                </artifactItems>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+
+        <plugin>
+          <artifactId>maven-resources-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>copy-interpreter-setting</id>
+              <phase>package</phase>
+              <goals>
+                <goal>resources</goal>
+              </goals>
+              <configuration>
+                <outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>

interpreter/lib/python/mpl_config.py

@@ -71,7 +71,11 @@ def _on_config_change():
     supported_formats = _config['supported_formats']
     if fmt not in supported_formats:
         raise ValueError("Unsupported format %s" %fmt)
-    matplotlib.rcParams['savefig.format'] = fmt
+
+    if matplotlib.__version__ < '1.2.0':
+        matplotlib.rcParams.update({'savefig.format': fmt})
+    else:
+        matplotlib.rcParams['savefig.format'] = fmt
     
     # Interactive mode
     interactive = _config['interactive']
@@ -80,6 +84,8 @@ def _on_config_change():
     
 def _init_config():
     dpi = matplotlib.rcParams['figure.dpi']
+    if matplotlib.__version__ < '1.2.0':
+        matplotlib.rcParams.update({'savefig.format': 'png'})
     fmt = matplotlib.rcParams['savefig.format']
     width, height = matplotlib.rcParams['figure.figsize']
     fontsize = matplotlib.rcParams['font.size']

jdbc/pom.xml

@@ -21,9 +21,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -34,6 +35,7 @@
   
   <properties>
     <!--library versions-->
+    <interpreter.name>jdbc</interpreter.name>
     <postgresql.version>9.4-1201-jdbc41</postgresql.version>
     <hadoop.common.version>2.7.2</hadoop.common.version>
     <h2.version>1.4.190</h2.version>
@@ -158,54 +160,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

@@ -38,14 +38,13 @@ import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
 import org.apache.commons.dbcp2.PoolableConnectionFactory;
 import org.apache.commons.dbcp2.PoolingDriver;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.commons.pool2.ObjectPool;
 import org.apache.commons.pool2.impl.GenericObjectPool;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -171,7 +170,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
   @Override
   public void open() {
     super.open();
-    for (String propertyKey : property.stringPropertyNames()) {
+    for (String propertyKey : properties.stringPropertyNames()) {
       logger.debug("propertyKey: {}", propertyKey);
       String[] keyValue = propertyKey.split("\\.", 2);
       if (2 == keyValue.length) {
@@ -184,7 +183,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
           prefixProperties = new Properties();
           basePropretiesMap.put(keyValue[0].trim(), prefixProperties);
         }
-        prefixProperties.put(keyValue[1].trim(), property.getProperty(propertyKey));
+        prefixProperties.put(keyValue[1].trim(), getProperty(propertyKey));
       }
     }
 
@@ -210,8 +209,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
 
 
   protected boolean isKerboseEnabled() {
-    if (!isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
-      UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
+    if (!isEmpty(getProperty("zeppelin.jdbc.auth.type"))) {
+      UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(properties);
       if (authType.equals(KERBEROS)) {
         return true;
       }
@@ -278,11 +277,17 @@ public class JDBCInterpreter extends KerberosInterpreter {
   }
 
   private void initConnectionPoolMap() {
-    for (JDBCUserConfigurations configurations : jdbcUserConfigurationsMap.values()) {
+    for (String key : jdbcUserConfigurationsMap.keySet()) {
       try {
+        closeDBPool(key, DEFAULT_KEY);
+      } catch (SQLException e) {
+        logger.error("Error while closing database pool.", e);
+      }
+      try {
+        JDBCUserConfigurations configurations = jdbcUserConfigurationsMap.get(key);
         configurations.initConnectionPoolMap();
-      } catch (Exception e) {
-        logger.error("Error while closing initConnectionPoolMap...", e);
+      } catch (SQLException e) {
+        logger.error("Error while closing initConnectionPoolMap.", e);
       }
     }
   }
@@ -349,7 +354,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
   }
 
   private void setUserProperty(String propertyKey, InterpreterContext interpreterContext)
-      throws SQLException, IOException {
+      throws SQLException, IOException, InterpreterException {
 
     String user = interpreterContext.getAuthenticationInfo().getUser();
 
@@ -417,18 +422,19 @@ public class JDBCInterpreter extends KerberosInterpreter {
     final Properties properties = jdbcUserConfigurations.getPropertyMap(propertyKey);
     final String url = properties.getProperty(URL_KEY);
 
-    if (isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
+    if (isEmpty(getProperty("zeppelin.jdbc.auth.type"))) {
       connection = getConnectionFromPool(url, user, propertyKey, properties);
     } else {
-      UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
+      UserGroupInformation.AuthenticationMethod authType =
+          JDBCSecurityImpl.getAuthtype(getProperties());
 
       final String connectionUrl = appendProxyUserToURL(url, user, propertyKey);
 
-      JDBCSecurityImpl.createSecureConfiguration(property, authType);
+      JDBCSecurityImpl.createSecureConfiguration(getProperties(), authType);
       switch (authType) {
         case KERBEROS:
           if (user == null || "false".equalsIgnoreCase(
-              property.getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
+              getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
             connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties);
           } else {
             if (basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) {
@@ -490,7 +496,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
     return connectionUrl.toString();
   }
 
-  private String getPassword(Properties properties) throws IOException {
+  private String getPassword(Properties properties) throws IOException, InterpreterException {
     if (isNotEmpty(properties.getProperty(PASSWORD_KEY))) {
       return properties.getProperty(PASSWORD_KEY);
     } else if (isNotEmpty(properties.getProperty(JDBC_JCEKS_FILE))
@@ -518,7 +524,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
     return null;
   }
 
-  private String getResults(ResultSet resultSet, boolean isTableType)
+  private String getResults(ResultSet resultSet, boolean isTableType, MutableBoolean isComplete)
       throws SQLException {
     ResultSetMetaData md = resultSet.getMetaData();
     StringBuilder msg;
@@ -537,7 +543,11 @@ public class JDBCInterpreter extends KerberosInterpreter {
     msg.append(NEWLINE);
 
     int displayRowCount = 0;
-    while (displayRowCount < getMaxResult() && resultSet.next()) {
+    while (resultSet.next()) {
+      if (displayRowCount >= getMaxResult()) {
+        isComplete.setValue(false);
+        break;
+      }
       for (int i = 1; i < md.getColumnCount() + 1; i++) {
         Object resultObject;
         String resultValue;
@@ -653,7 +663,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
 
   private InterpreterResult executeSql(String propertyKey, String sql,
       InterpreterContext interpreterContext) {
-    Connection connection;
+    Connection connection = null;
     Statement statement;
     ResultSet resultSet = null;
     String paragraphId = interpreterContext.getParagraphId();
@@ -668,11 +678,21 @@ public class JDBCInterpreter extends KerberosInterpreter {
     InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
     try {
       connection = getConnection(propertyKey, interpreterContext);
-      if (connection == null) {
-        return new InterpreterResult(Code.ERROR, "Prefix not found.");
+    } catch (Exception e) {
+      String errorMsg = Throwables.getStackTraceAsString(e);
+      try {
+        closeDBPool(user, propertyKey);
+      } catch (SQLException e1) {
+        logger.error("Cannot close DBPool for user, propertyKey: " + user + propertyKey, e1);
       }
+      interpreterResult.add(errorMsg);
+      return new InterpreterResult(Code.ERROR, interpreterResult.message());
+    }
+    if (connection == null) {
+      return new InterpreterResult(Code.ERROR, "Prefix not found.");
+    }
 
-
+    try {
       List<String> sqlArray;
       if (splitQuery) {
         sqlArray = splitSqlQueries(sql);
@@ -706,10 +726,11 @@ public class JDBCInterpreter extends KerberosInterpreter {
               interpreterResult.add(InterpreterResult.Type.TEXT,
                   "Query executed successfully.");
             } else {
+              MutableBoolean isComplete = new MutableBoolean(true);
               String results = getResults(resultSet,
-                  !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE));
+                  !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE), isComplete);
               interpreterResult.add(results);
-              if (resultSet.next()) {
+              if (!isComplete.booleanValue()) {
                 interpreterResult.add(ResultMessages.getExceedsLimitRowsMessage(getMaxResult(),
                     String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY)));
               }
@@ -734,6 +755,12 @@ public class JDBCInterpreter extends KerberosInterpreter {
           }
         }
       }
+    } catch (Throwable e) {
+      logger.error("Cannot run " + sql, e);
+      String errorMsg = Throwables.getStackTraceAsString(e);
+      interpreterResult.add(errorMsg);
+      return new InterpreterResult(Code.ERROR, interpreterResult.message());
+    } finally {
       //In case user ran an insert/update/upsert statement
       if (connection != null) {
         try {
@@ -744,16 +771,6 @@ public class JDBCInterpreter extends KerberosInterpreter {
         } catch (SQLException e) { /*ignored*/ }
       }
       getJDBCConfiguration(user).removeStatement(paragraphId);
-    } catch (Throwable e) {
-      logger.error("Cannot run " + sql, e);
-      String errorMsg = Throwables.getStackTraceAsString(e);
-      try {
-        closeDBPool(user, propertyKey);
-      } catch (SQLException e1) {
-        logger.error("Cannot close DBPool for user, propertyKey: " + user + propertyKey, e1);
-      }
-      interpreterResult.add(errorMsg);
-      return new InterpreterResult(Code.ERROR, interpreterResult.message());
     }
     return interpreterResult;
   }
@@ -832,7 +849,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
 
   @Override
   public List<InterpreterCompletion> completion(String buf, int cursor,
-      InterpreterContext interpreterContext) {
+      InterpreterContext interpreterContext) throws InterpreterException {
     List<InterpreterCompletion> candidates = new ArrayList<>();
     String propertyKey = getPropertyKey(buf);
     String sqlCompleterKey =

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java

@@ -48,12 +48,6 @@ public class JDBCUserConfigurations {
   }
 
   public void initConnectionPoolMap() throws SQLException {
-    Iterator<String> it = poolingDriverMap.keySet().iterator();
-    while (it.hasNext()) {
-      String driverName = it.next();
-      poolingDriverMap.get(driverName).closePool(driverName);
-      it.remove();
-    }
     poolingDriverMap.clear();
     isSuccessful.clear();
   }

jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java

@@ -179,7 +179,8 @@ public class SqlCompleter {
 
 
   private static void fillTableNames(String schema, DatabaseMetaData meta, Set<String> tables) {
-    try (ResultSet tbls = meta.getTables(schema, schema, "%", null)) {
+    try (ResultSet tbls = meta.getTables(schema, schema, "%",
+        new String[]{"TABLE", "VIEW", "ALIAS", "SYNONYM", "GLOBAL TEMPORARY", "LOCAL TEMPORARY"})) {
       while (tbls.next()) {
         String table = tbls.getString("TABLE_NAME");
         tables.add(table);

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -37,6 +37,7 @@ import java.util.Properties;
 
 import org.apache.zeppelin.completer.CompletionType;
 import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.FIFOScheduler;
@@ -349,7 +350,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
   }
 
   @Test
-  public void testAutoCompletion() throws SQLException, IOException {
+  public void testAutoCompletion() throws SQLException, IOException, InterpreterException {
     Properties properties = new Properties();
     properties.setProperty("common.max_count", "1000");
     properties.setProperty("common.max_retry", "3");

kylin/pom.xml

@@ -21,9 +21,10 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>zeppelin</artifactId>
+        <artifactId>interpreter-parent</artifactId>
         <groupId>org.apache.zeppelin</groupId>
         <version>0.8.0-SNAPSHOT</version>
+        <relativePath>../interpreter-parent</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -33,6 +34,10 @@
     <version>0.8.0-SNAPSHOT</version>
     <name>Zeppelin: Kylin interpreter</name>
 
+    <properties>
+        <interpreter.name>kylin</interpreter.name>
+    </properties>
+
     <dependencies>
 
         <dependency>
@@ -61,57 +66,14 @@
         <plugins>
             <plugin>
                 <artifactId>maven-enforcer-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>enforce</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
             </plugin>
-
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>copy-dependencies</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy-dependencies</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>copy-artifact</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                            <artifactItems>
-                                <artifactItem>
-                                    <groupId>${project.groupId}</groupId>
-                                    <artifactId>${project.artifactId}</artifactId>
-                                    <version>${project.version}</version>
-                                    <type>${project.packaging}</type>
-                                </artifactItem>
-                            </artifactItems>
-                        </configuration>
-                    </execution>
-                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
             </plugin>
         </plugins>
     </build>
 
-
 </project>

lens/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -34,6 +34,7 @@
   
   <properties>
     <!--library versions-->
+    <interpreter.name>lens</interpreter.name>
     <lens.version>2.5.0-beta</lens.version>
     <spring-shell.version>1.1.0.RELEASE</spring-shell.version>
     <hadoop-common.version>2.4.0</hadoop-common.version>
@@ -163,69 +164,13 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
       </plugin>
-
-	<plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-clean-plugin</artifactId>
-          <configuration>
-            <filesets>
-              <fileset>
-                <directory>${basedir}/../interpreter/lens</directory>
-                <followSymlinks>false</followSymlinks>
-              </fileset>
-            </filesets>
-          </configuration>
-        </plugin>
-
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
+      </plugin>
     </plugins>
   </build>
 

livy/pom.xml

@@ -19,544 +19,248 @@
 <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>interpreter-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.8.0-SNAPSHOT</version>
+        <relativePath>../interpreter-parent</relativePath>
+    </parent>
 
-  <parent>
-    <artifactId>zeppelin</artifactId>
     <groupId>org.apache.zeppelin</groupId>
+    <artifactId>zeppelin-livy</artifactId>
+    <packaging>jar</packaging>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
-  </parent>
+    <name>Zeppelin: Livy interpreter</name>
 
-  <groupId>org.apache.zeppelin</groupId>
-  <artifactId>zeppelin-livy</artifactId>
-  <packaging>jar</packaging>
-  <version>0.8.0-SNAPSHOT</version>
-  <name>Zeppelin: Livy interpreter</name>
+    <properties>
+        <!--library versions-->
+        <interpreter.name>livy</interpreter.name>
+        <commons.exec.version>1.3</commons.exec.version>
+        <spring.web.version>4.3.0.RELEASE</spring.web.version>
+        <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
 
-  <properties>
-    <!--library versions-->
-    <commons.exec.version>1.3</commons.exec.version>
-    <spring.web.version>4.3.0.RELEASE</spring.web.version>
-    <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
-
-    <!--test library versions-->
-    <livy.version>0.3.0</livy.version>
-    <spark.version>2.1.0</spark.version>
-    <hadoop.version>2.6.0</hadoop.version>
-    <!--plugin versions-->
-    <plugin.failsafe.version>2.16</plugin.failsafe.version>
-    <plugin.antrun.version>1.8</plugin.antrun.version>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>zeppelin-interpreter</artifactId>
-      <version>${project.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-exec</artifactId>
-      <version>${commons.exec.version}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.code.gson</groupId>
-      <artifactId>gson</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.springframework.security.kerberos</groupId>
-      <artifactId>spring-security-kerberos-client</artifactId>
-      <version>${spring.security.kerberosclient}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>org.springframework</groupId>
-      <artifactId>spring-web</artifactId>
-      <version>${spring.web.version}</version>
-    </dependency>
-
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.cloudera.livy</groupId>
-      <artifactId>livy-integration-test</artifactId>
-      <version>${livy.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.xerial.snappy</groupId>
-          <artifactId>snappy-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-core_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-repl_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-auth</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-hdfs</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-client</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-client</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-tests</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>com.cloudera.livy</groupId>
-      <artifactId>livy-test-lib</artifactId>
-      <version>${livy.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.xerial.snappy</groupId>
-          <artifactId>snappy-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-core_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-sql_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-hive_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-repl_${scala.binary.version}</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.spark</groupId>
-          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.esotericsoftware</groupId>
-          <artifactId>kryo-shaded</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-repl_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-web-proxy</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-client</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-api</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-tests</artifactId>
-      <classifier>tests</classifier>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-
-  <repositories>
-    <repository>
-      <id>ossrh</id>
-      <name>ossrh repository</name>
-      <url>https://oss.sonatype.org/content/repositories/releases/</url>
-      <releases>
-        <enabled>true</enabled>
-      </releases>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-    </repository>
-  </repositories>
-
-  <build>
-    <plugins>
-      <plugin>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>enforce</id>
-            <phase>none</phase>
-          </execution>
-        </executions>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/livy
-              </outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/livy
-              </outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-failsafe-plugin</artifactId>
-        <version>${plugin.failsafe.version}</version>
-        <executions>
-          <execution>
-            <goals>
-              <goal>integration-test</goal>
-              <goal>verify</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <systemPropertyVariables>
-            <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
-          </systemPropertyVariables>
-          <environmentVariables>
-            <LIVY_SPARK_SCALA_VERSION>${scala.binary.version}</LIVY_SPARK_SCALA_VERSION>
-            <LIVY_LOG_DIR>${project.build.directory}/tmp</LIVY_LOG_DIR>
-          </environmentVariables>
-          <argLine>-Xmx2048m</argLine>
-        </configuration>
-      </plugin>
-
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <version>${plugin.antrun.version}</version>
-        <executions>
-          <!-- Cleans up files that tests append to (because we have two test plugins). -->
-          <execution>
-            <id>pre-test-clean</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <configuration>
-              <target>
-                <delete file="${project.build.directory}/unit-tests.log"
-                        quiet="true"/>
-                <delete file="${project.build.directory}/jacoco.exec" quiet="true"/>
-                <delete dir="${project.build.directory}/tmp" quiet="true"/>
-              </target>
-            </configuration>
-          </execution>
-          <!-- Create the temp directory to be  used by tests. -->
-          <execution>
-            <id>create-tmp-dir</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <configuration>
-              <target>
-                <mkdir dir="${project.build.directory}/tmp"/>
-              </target>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-
-  <profiles>
-    <profile>
-      <id>livy-0.3</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
-      <properties>
-        <livy.version>0.3.0</livy.version>
+        <!--test library versions-->
+        <livy.version>0.4.0-incubating</livy.version>
         <spark.version>2.1.0</spark.version>
-        <hadoop.version>2.6.0</hadoop.version>
-      </properties>
-      <dependencies>
-        <dependency>
-          <groupId>com.cloudera.livy</groupId>
-          <artifactId>livy-core_${scala.binary.version}</artifactId>
-          <version>0.3.0</version>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.xerial.snappy</groupId>
-              <artifactId>snappy-java</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-core_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
-    </profile>
+        <!--plugin versions-->
+        <plugin.failsafe.version>2.16</plugin.failsafe.version>
+        <plugin.antrun.version>1.8</plugin.antrun.version>
+    </properties>
 
-    <profile>
-      <id>livy-0.2</id>
-      <properties>
-        <livy.version>0.2.0</livy.version>
-        <spark.version>1.6.2</spark.version>
-        <hadoop.version>2.6.0</hadoop.version>
-        <scala.binary.version>2.10</scala.binary.version>
-      </properties>
-      <dependencies>
+    <dependencies>
         <dependency>
-          <groupId>com.cloudera.livy</groupId>
-          <artifactId>livy-core</artifactId>
-          <version>0.2.0</version>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.xerial.snappy</groupId>
-              <artifactId>snappy-java</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-core_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-repl_${scala.binary.version}</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.spark</groupId>
-              <artifactId>spark-yarn_${scala.binary.version}</artifactId>
-            </exclusion>
-          </exclusions>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>zeppelin-interpreter</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-compiler</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-library</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.scala-lang</groupId>
+                    <artifactId>scala-reflect</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
+
+        <dependency>
+            <groupId>org.apache.livy</groupId>
+            <artifactId>livy-integration-test</artifactId>
+            <version>${livy.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-core_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-sql_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-hive_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-repl_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.livy</groupId>
+            <artifactId>livy-test-lib</artifactId>
+            <version>${livy.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-core_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-sql_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-hive_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-repl_${scala.binary.version}</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.spark</groupId>
+                    <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-exec</artifactId>
+            <version>${commons.exec.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.security.kerberos</groupId>
+            <artifactId>spring-security-kerberos-client</artifactId>
+            <version>${spring.security.kerberosclient}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-web</artifactId>
+            <version>${spring.web.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <version>${plugin.failsafe.version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <systemPropertyVariables>
+                        <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
+                    </systemPropertyVariables>
+                    <environmentVariables>
+                        <LIVY_SPARK_SCALA_VERSION>${scala.binary.version}</LIVY_SPARK_SCALA_VERSION>
+                        <LIVY_LOG_DIR>${project.build.directory}/tmp</LIVY_LOG_DIR>
+                    </environmentVariables>
+                    <argLine>-Xmx2048m</argLine>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>${plugin.antrun.version}</version>
+                <executions>
+                    <!-- Cleans up files that tests append to (because we have two test plugins). -->
+                    <execution>
+                        <id>pre-test-clean</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <delete file="${project.build.directory}/unit-tests.log"
+                                        quiet="true"/>
+                                <delete file="${project.build.directory}/jacoco.exec" quiet="true"/>
+                                <delete dir="${project.build.directory}/tmp" quiet="true"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                    <!-- Create the temp directory to be  used by tests. -->
+                    <execution>
+                        <id>create-tmp-dir</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <mkdir dir="${project.build.directory}/tmp"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java

@@ -41,6 +41,7 @@ import org.apache.zeppelin.interpreter.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpEntity;
+import org.springframework.http.MediaType;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
@@ -61,6 +62,8 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 
 /**
@@ -79,6 +82,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
   protected boolean displayAppInfo;
   protected LivyVersion livyVersion;
   private RestTemplate restTemplate;
+  private Map<String, String> customHeaders = new HashMap<>();
 
   Set<Object> paragraphsToCancel = Collections.newSetFromMap(
       new ConcurrentHashMap<Object, Boolean>());
@@ -95,12 +99,39 @@ public abstract class BaseLivyInterpreter extends Interpreter {
     this.pullStatusInterval = Integer.parseInt(
         property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
     this.restTemplate = createRestTemplate();
+    if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) {
+      String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";");
+      for (String header : headers) {
+        String[] splits = header.split(":", -1);
+        if (splits.length != 2) {
+          throw new RuntimeException("Invalid format of http headers: " + header +
+              ", valid http header format is HEADER_NAME:HEADER_VALUE");
+        }
+        customHeaders.put(splits[0].trim(), envSubstitute(splits[1].trim()));
+      }
+    }
+  }
+
+  private String envSubstitute(String value) {
+    String newValue = new String(value);
+    Pattern pattern = Pattern.compile("\\$\\{(.*)\\}");
+    Matcher matcher = pattern.matcher(value);
+    while (matcher.find()) {
+      String env = matcher.group(1);
+      newValue = newValue.replace("${" + env + "}", System.getenv(env));
+    }
+    return newValue;
+  }
+
+  // only for testing
+  Map<String, String> getCustomHeaders() {
+    return customHeaders;
   }
 
   public abstract String getSessionKind();
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
     try {
       initLivySession();
     } catch (LivyException e) {
@@ -197,7 +228,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
       throws LivyException {
     try {
       Map<String, String> conf = new HashMap<>();
-      for (Map.Entry<Object, Object> entry : property.entrySet()) {
+      for (Map.Entry<Object, Object> entry : getProperties().entrySet()) {
         if (entry.getKey().toString().startsWith("livy.spark.") &&
             !entry.getValue().toString().isEmpty())
           conf.put(entry.getKey().toString().substring(5), entry.getValue().toString());
@@ -427,15 +458,15 @@ public abstract class BaseLivyInterpreter extends Interpreter {
 
 
   private RestTemplate createRestTemplate() {
-    String keytabLocation = property.getProperty("zeppelin.livy.keytab");
-    String principal = property.getProperty("zeppelin.livy.principal");
+    String keytabLocation = getProperty("zeppelin.livy.keytab");
+    String principal = getProperty("zeppelin.livy.principal");
     boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
         StringUtils.isNotEmpty(principal);
 
     HttpClient httpClient = null;
     if (livyURL.startsWith("https:")) {
-      String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
-      String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword");
+      String keystoreFile = getProperty("zeppelin.livy.ssl.trustStore");
+      String password = getProperty("zeppelin.livy.ssl.trustStorePassword");
       if (StringUtils.isBlank(keystoreFile)) {
         throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl");
       }
@@ -520,8 +551,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
     targetURL = livyURL + targetURL;
     LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData);
     HttpHeaders headers = new HttpHeaders();
-    headers.add("Content-Type", "application/json");
+    headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE);
     headers.add("X-Requested-By", "zeppelin");
+    for (Map.Entry<String, String> entry : customHeaders.entrySet()) {
+      headers.add(entry.getKey(), entry.getValue());
+    }
     ResponseEntity<String> response = null;
     try {
       if (method.equals("POST")) {

livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java

@@ -59,7 +59,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
   }
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
     this.sparkInterpreter = getSparkInterpreter();
     // As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession
     // to judge whether it is using spark2.
@@ -93,7 +93,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
     }
   }
 
-  private LivySparkInterpreter getSparkInterpreter() {
+  private LivySparkInterpreter getSparkInterpreter() throws InterpreterException {
     LazyOpenInterpreter lazy = null;
     LivySparkInterpreter spark = null;
     Interpreter p = getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());

livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java

@@ -18,8 +18,8 @@
 package org.apache.zeppelin.livy;
 
 
-import com.cloudera.livy.test.framework.Cluster;
-import com.cloudera.livy.test.framework.Cluster$;
+import org.apache.livy.test.framework.Cluster;
+import org.apache.livy.test.framework.Cluster$;
 import org.apache.commons.io.IOUtils;
 import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.user.AuthenticationInfo;
@@ -28,6 +28,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Properties;
 
@@ -74,8 +75,9 @@ public class LivyInterpreterIT {
     return true;
   }
 
-  @Test
-  public void testSparkInterpreterRDD() {
+
+//  @Test
+  public void testSparkInterpreterRDD() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -194,8 +196,9 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testSparkInterpreterDataFrame() {
+
+//  @Test
+  public void testSparkInterpreterDataFrame() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -282,8 +285,8 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testSparkSQLInterpreter() {
+//  @Test
+  public void testSparkSQLInterpreter() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -317,8 +320,8 @@ public class LivyInterpreterIT {
   }
 
 
-  @Test
-  public void testSparkSQLCancellation() {
+//  @Test
+  public void testSparkSQLCancellation() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -398,8 +401,8 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testStringWithTruncation() {
+//  @Test
+  public void testStringWithTruncation() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -458,8 +461,9 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testStringWithoutTruncation() {
+
+//  @Test
+  public void testStringWithoutTruncation() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -524,7 +528,7 @@ public class LivyInterpreterIT {
   }
 
   @Test
-  public void testPySparkInterpreter() throws LivyException {
+  public void testPySparkInterpreter() throws LivyException, InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -555,6 +559,16 @@ public class LivyInterpreterIT {
       assertTrue(result.message().get(0).getData().contains("Traceback"));
     }
 
+    // test utf-8 Encoding
+    try {
+      String utf8Str = "你你你你你你好";
+      InterpreterResult result = pysparkInterpreter.interpret("print(\""+utf8Str+"\")", context);
+      assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+      assertTrue(result.message().get(0).getData().contains(utf8Str));
+    }catch (Exception e) {
+      e.printStackTrace();
+    }
+
     try {
       InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -633,8 +647,8 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testSparkInterpreterWithDisplayAppInfo() {
+//  @Test
+  public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -672,8 +686,8 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testSparkRInterpreter() throws LivyException {
+//  @Test
+  public void testSparkRInterpreter() throws LivyException, InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -744,8 +758,8 @@ public class LivyInterpreterIT {
     }
   }
 
-  @Test
-  public void testLivyTutorialNote() throws IOException {
+//  @Test
+  public void testLivyTutorialNote() throws IOException, InterpreterException {
     if (!checkPreCondition()) {
       return;
     }

livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java

@@ -39,9 +39,17 @@ public class LivySQLInterpreterTest {
     properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
     properties.setProperty("zeppelin.livy.session.create_timeout", "120");
     properties.setProperty("zeppelin.livy.spark.sql.maxResult", "3");
+    properties.setProperty("zeppelin.livy.http.headers", "HEADER_1: VALUE_1_${HOME}");
     sqlInterpreter = new LivySparkSQLInterpreter(properties);
   }
 
+  @Test
+  public void testHttpHeaders() {
+    assertEquals(1, sqlInterpreter.getCustomHeaders().size());
+    assertTrue(sqlInterpreter.getCustomHeaders().get("HEADER_1").startsWith("VALUE_1_"));
+    assertNotEquals("VALUE_1_${HOME}", sqlInterpreter.getCustomHeaders().get("HEADER_1"));
+  }
+
   @Test
   public void testParseSQLOutput() {
     // Empty sql output

markdown/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -33,6 +33,7 @@
   <name>Zeppelin: Markdown interpreter</name>
 
   <properties>
+    <interpreter.name>md</interpreter.name>
     <commons.lang3.version>3.4</commons.lang3.version>
     <markdown4j.version>2.2-cj-1.0</markdown4j.version>
     <pegdown.version>1.6.0</pegdown.version>
@@ -85,54 +86,12 @@
     <plugins>
       <plugin>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <executions>
-          <execution> 
-            <id>enforce</id> 
-            <phase>none</phase> 
-          </execution>
-        </executions>
       </plugin>
-
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/md</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/md</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>              
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
     </plugins>
   </build>

neo4j/pom.xml

@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+  
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>zeppelin</artifactId>
+    <groupId>org.apache.zeppelin</groupId>
+    <version>0.8.0-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <groupId>org.apache.zeppelin</groupId>
+  <artifactId>zeppelin-neo4j</artifactId>
+  <packaging>jar</packaging>
+  <version>0.8.0-SNAPSHOT</version>
+  <name>Zeppelin: Neo4j interpreter</name>
+  
+  <properties>
+  	<neo4j.driver.version>1.4.3</neo4j.driver.version>
+  	<test.neo4j.kernel.version>3.2.3</test.neo4j.kernel.version>
+  	<neo4j.version>3.2.3</neo4j.version>
+  	<jackson.version>2.8.9</jackson.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zeppelin-interpreter</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.neo4j.driver</groupId>
+      <artifactId>neo4j-java-driver</artifactId>
+      <version>${neo4j.driver.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.neo4j.test</groupId>
+      <artifactId>neo4j-harness</artifactId>
+      <version>${neo4j.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.3.1</version>            
+        <executions> 
+          <execution> 
+            <id>enforce</id> 
+            <phase>none</phase> 
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>2.8</version>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/../../interpreter/neo4j</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+              <overWriteIfNewer>true</overWriteIfNewer>
+              <includeScope>runtime</includeScope>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-artifact</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/../../interpreter/neo4j</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+              <overWriteIfNewer>true</overWriteIfNewer>
+              <includeScope>runtime</includeScope>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>${project.artifactId}</artifactId>
+                  <version>${project.version}</version>
+                  <type>${project.packaging}</type>
+                </artifactItem>
+              </artifactItems>              
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jConnectionManager.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.graph.neo4j;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.neo4j.driver.v1.AuthToken;
+import org.neo4j.driver.v1.AuthTokens;
+import org.neo4j.driver.v1.Config;
+import org.neo4j.driver.v1.Driver;
+import org.neo4j.driver.v1.GraphDatabase;
+import org.neo4j.driver.v1.Session;
+import org.neo4j.driver.v1.StatementResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Neo4j connection manager for Zeppelin.
+ */
+public class Neo4jConnectionManager {
+  static final Logger LOGGER = LoggerFactory.getLogger(Neo4jConnectionManager.class);
+  
+  public static final String NEO4J_SERVER_URL = "neo4j.url";
+  public static final String NEO4J_AUTH_TYPE = "neo4j.auth.type";
+  public static final String NEO4J_AUTH_USER = "neo4j.auth.user";
+  public static final String NEO4J_AUTH_PASSWORD = "neo4j.auth.password";
+  public static final String NEO4J_MAX_CONCURRENCY = "neo4j.max.concurrency";
+
+  private static final Pattern PROPERTY_PATTERN = Pattern.compile("\\{\\w+\\}");
+  private static final String REPLACE_CURLY_BRACKETS = "\\{|\\}";
+
+  private static final Pattern $_PATTERN = Pattern.compile("\\$\\w+\\}");
+  private static final String REPLACE_$ = "\\$";
+
+  private Driver driver = null;
+
+  private final String neo4jUrl;
+
+  private final Config config;
+
+  private final AuthToken authToken;
+
+  /**
+   * 
+   * Enum type for the AuthToken 
+   *
+   */
+  public enum Neo4jAuthType {NONE, BASIC}
+
+  public Neo4jConnectionManager(Properties properties) {
+    this.neo4jUrl = properties.getProperty(NEO4J_SERVER_URL);
+    this.config = Config.build()
+          .withMaxIdleSessions(Integer.parseInt(properties.getProperty(NEO4J_MAX_CONCURRENCY)))
+          .toConfig();
+    String authType = properties.getProperty(NEO4J_AUTH_TYPE);
+    switch (Neo4jAuthType.valueOf(authType.toUpperCase())) {
+      case BASIC:
+        String username = properties.getProperty(NEO4J_AUTH_USER);
+        String password = properties.getProperty(NEO4J_AUTH_PASSWORD);
+        this.authToken = AuthTokens.basic(username, password);
+        break;
+      case NONE:
+        LOGGER.debug("Creating NONE authentication");
+        this.authToken = AuthTokens.none();
+        break;
+      default:
+        throw new RuntimeException("Neo4j authentication type not supported");
+    }
+  }
+
+  private Driver getDriver() {
+    if (driver == null) {
+      driver = GraphDatabase.driver(this.neo4jUrl, this.authToken, this.config);
+    }
+    return driver;
+  }
+
+  public void open() {
+    getDriver();
+  }
+
+  public void close() {
+    getDriver().close();
+  }
+
+  private Session getSession() {
+    return getDriver().session();
+  }
+
+  public StatementResult execute(String cypherQuery,
+      InterpreterContext interpreterContext) {
+    Map<String, Object> params = new HashMap<>();
+    if (interpreterContext != null) {
+      ResourcePool resourcePool = interpreterContext.getResourcePool();
+      Set<String> keys = extractParams(cypherQuery, PROPERTY_PATTERN, REPLACE_CURLY_BRACKETS);
+      keys.addAll(extractParams(cypherQuery, $_PATTERN, REPLACE_$));
+      for (String key : keys) {
+        Resource resource = resourcePool.get(key);
+        if (resource != null) {
+          params.put(key, resource.get());
+        }
+      }
+    }
+    LOGGER.debug("Executing cypher query {} with params {}", cypherQuery, params);
+    StatementResult result;
+    try (Session session = getSession()) {
+      result = params.isEmpty()
+            ? getSession().run(cypherQuery) : getSession().run(cypherQuery, params);
+    }
+    return result;
+  }
+
+  public StatementResult execute(String cypherQuery) {
+    return execute(cypherQuery, null);
+  }
+
+  private Set<String> extractParams(String cypherQuery, Pattern pattern, String replaceChar) {
+    Matcher matcher = pattern.matcher(cypherQuery);
+    Set<String> keys = new HashSet<>();
+    while (matcher.find()) {
+      keys.add(matcher.group().replaceAll(replaceChar, StringUtils.EMPTY));
+    }
+    return keys;
+  }
+
+}

neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreter.java

@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.graph.neo4j;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.graph.neo4j.utils.Neo4jConversionUtils;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.graph.GraphResult;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.neo4j.driver.internal.types.InternalTypeSystem;
+import org.neo4j.driver.internal.util.Iterables;
+import org.neo4j.driver.v1.Record;
+import org.neo4j.driver.v1.StatementResult;
+import org.neo4j.driver.v1.Value;
+import org.neo4j.driver.v1.types.Node;
+import org.neo4j.driver.v1.types.Relationship;
+import org.neo4j.driver.v1.types.TypeSystem;
+import org.neo4j.driver.v1.util.Pair;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * Neo4j interpreter for Zeppelin.
+ */
+public class Neo4jCypherInterpreter extends Interpreter {
+  private static final String TABLE = "%table";
+  public static final String NEW_LINE = "\n";
+  public static final String TAB = "\t";
+
+  private static final String MAP_KEY_TEMPLATE = "%s.%s";
+
+  private Map<String, String> labels;
+
+  private Set<String> types;
+  
+  private final Neo4jConnectionManager neo4jConnectionManager;
+  
+  private final ObjectMapper jsonMapper = new ObjectMapper();
+
+  public Neo4jCypherInterpreter(Properties properties) {
+    super(properties);
+    this.neo4jConnectionManager = new Neo4jConnectionManager(properties);
+  }
+
+  @Override
+  public void open() {
+    this.neo4jConnectionManager.open();
+  }
+
+  @Override
+  public void close() {
+    this.neo4jConnectionManager.close();
+  }
+
+  public Map<String, String> getLabels(boolean refresh) {
+    if (labels == null || refresh) {
+      Map<String, String> old = labels == null ?
+          new LinkedHashMap<String, String>() : new LinkedHashMap<>(labels);
+      labels = new LinkedHashMap<>();
+      StatementResult result = this.neo4jConnectionManager.execute("CALL db.labels()");
+      Set<String> colors = new HashSet<>();
+      while (result.hasNext()) {
+        Record record = result.next();
+        String label = record.get("label").asString();
+        String color = old.get(label);
+        while (color == null || colors.contains(color)) {
+          color = Neo4jConversionUtils.getRandomLabelColor();
+        }
+        colors.add(color);
+        labels.put(label, color);
+      }
+    }
+    return labels;
+  }
+
+  private Set<String> getTypes(boolean refresh) {
+    if (types == null || refresh) {
+      types = new HashSet<>();
+      StatementResult result = this.neo4jConnectionManager.execute("CALL db.relationshipTypes()");
+      while (result.hasNext()) {
+        Record record = result.next();
+        types.add(record.get("relationshipType").asString());
+      }
+    }
+    return types;
+  }
+
+  @Override
+  public InterpreterResult interpret(String cypherQuery, InterpreterContext interpreterContext) {
+    logger.info("Opening session");
+    if (StringUtils.isBlank(cypherQuery)) {
+      return new InterpreterResult(Code.SUCCESS);
+    }
+    try {
+      StatementResult result = this.neo4jConnectionManager.execute(cypherQuery,
+              interpreterContext);
+      Set<Node> nodes = new HashSet<>();
+      Set<Relationship> relationships = new HashSet<>();
+      List<String> columns = new ArrayList<>();
+      List<List<String>> lines = new ArrayList<List<String>>();
+      while (result.hasNext()) {
+        Record record = result.next();
+        List<Pair<String, Value>> fields = record.fields();
+        List<String> line = new ArrayList<>();
+        for (Pair<String, Value> field : fields) {
+          if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.NODE())) {
+            nodes.add(field.value().asNode());
+          } else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.RELATIONSHIP())) {
+            relationships.add(field.value().asRelationship());
+          } else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.PATH())) {
+            nodes.addAll(Iterables.asList(field.value().asPath().nodes()));
+            relationships.addAll(Iterables.asList(field.value().asPath().relationships()));
+          } else {
+            setTabularResult(field.key(), field.value(), columns, line,
+                    InternalTypeSystem.TYPE_SYSTEM);
+          }
+        }
+        if (!line.isEmpty()) {
+          lines.add(line);
+        }
+      }
+      if (!nodes.isEmpty()) {
+        return renderGraph(nodes, relationships);
+      } else {
+        return renderTable(columns, lines);
+      }
+    } catch (Exception e) {
+      logger.error("Exception while interpreting cypher query", e);
+      return new InterpreterResult(Code.ERROR, e.getMessage());
+    }
+  }
+
+  private void setTabularResult(String key, Object obj, List<String> columns, List<String> line,
+      TypeSystem typeSystem) {
+    if (obj instanceof Value) {
+      Value value = (Value) obj;
+      if (value.hasType(typeSystem.MAP())) {
+        Map<String, Object> map = value.asMap();
+        for (Entry<String, Object> entry : map.entrySet()) {
+          setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(),
+                columns, line, typeSystem);
+        }
+      } else {
+        addValueToLine(key, columns, line, value);
+      }
+    } else if (obj instanceof Map) {
+      Map<String, Object> map = (Map<String, Object>) obj;
+      for (Entry<String, Object> entry : map.entrySet()) {
+        setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(),
+                columns, line, typeSystem);
+      }
+    } else {
+      addValueToLine(key, columns, line, obj);
+    }
+  }
+
+  private void addValueToLine(String key, List<String> columns, List<String> line, Object value) {
+    if (!columns.contains(key)) {
+      columns.add(key);
+    }
+    int position = columns.indexOf(key);
+    if (line.size() < columns.size()) {
+      for (int i = line.size(); i < columns.size(); i++) {
+        line.add(null);
+      }
+    }
+    if (value != null) {
+      if (value instanceof Value) {
+        Value val = (Value) value;
+        if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.LIST())) {
+          value = val.asList();
+        } else if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.MAP())) {
+          value = val.asMap();
+        }
+      }
+      if (value instanceof Collection) {
+        try {
+          value = jsonMapper.writer().writeValueAsString(value);
+        } catch (Exception ignored) {}
+      }
+    }
+    line.set(position, value == null ? null : value.toString());
+  }
+
+  private InterpreterResult renderTable(List<String> cols, List<List<String>> lines) {
+    logger.info("Executing renderTable method");
+    StringBuilder msg = null;
+    if (cols.isEmpty()) {
+      msg = new StringBuilder();
+    } else {
+      msg = new StringBuilder(TABLE);
+      msg.append(NEW_LINE);
+      msg.append(StringUtils.join(cols, TAB));
+      msg.append(NEW_LINE);
+      for (List<String> line : lines) {
+        if (line.size() < cols.size()) {
+          for (int i = line.size(); i < cols.size(); i++) {
+            line.add(null);
+          }
+        }
+        msg.append(StringUtils.join(line, TAB));
+        msg.append(NEW_LINE);
+      }
+    }
+    return new InterpreterResult(Code.SUCCESS, msg.toString());
+  }
+
+  private InterpreterResult renderGraph(Set<Node> nodes,
+      Set<Relationship> relationships) {
+    logger.info("Executing renderGraph method");
+    List<org.apache.zeppelin.tabledata.Node> nodesList = new ArrayList<>();
+    List<org.apache.zeppelin.tabledata.Relationship> relsList = new ArrayList<>();
+    for (Relationship rel : relationships) {
+      relsList.add(Neo4jConversionUtils.toZeppelinRelationship(rel));
+    }
+    Map<String, String> labels = getLabels(true);
+    for (Node node : nodes) {
+      nodesList.add(Neo4jConversionUtils.toZeppelinNode(node, labels));
+    }
+    return new GraphResult(Code.SUCCESS,
+        new GraphResult.Graph(nodesList, relsList, labels, getTypes(true), true));
+  }
+
+  @Override
+  public Scheduler getScheduler() {
+    return SchedulerFactory.singleton()
+        .createOrGetParallelScheduler(Neo4jCypherInterpreter.class.getName() + this.hashCode(),
+            Integer.parseInt(getProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY)));
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return 0;
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.SIMPLE;
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+  }
+
+}

neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/utils/Neo4jConversionUtils.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.graph.neo4j.utils;
+
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.neo4j.driver.v1.types.Node;
+import org.neo4j.driver.v1.types.Relationship;
+
+/**
+ * Neo4jConversionUtils
+ */
+public class Neo4jConversionUtils {
+  private Neo4jConversionUtils() {}
+  
+  private static final String[] LETTERS = "0123456789ABCDEF".split("");
+
+  public static final String COLOR_GREY = "#D3D3D3";
+  
+  public static org.apache.zeppelin.tabledata.Node toZeppelinNode(Node n,
+      Map<String, String> graphLabels) {
+    Set<String> labels = new LinkedHashSet<>();
+    String firstLabel = null;
+    for (String label : n.labels()) {
+      if (firstLabel == null) {
+        firstLabel = label;
+      }
+      labels.add(label);
+    }
+    return new org.apache.zeppelin.tabledata.Node(n.id(), n.asMap(),
+        labels);
+  }
+  
+  public static org.apache.zeppelin.tabledata.Relationship
+  toZeppelinRelationship(Relationship r) {
+    return new org.apache.zeppelin.tabledata.Relationship(r.id(), r.asMap(),
+        r.startNodeId(), r.endNodeId(), r.type());
+  }
+
+  public static String getRandomLabelColor() {
+    char[] color = new char[7];
+    color[0] = '#';
+    for (int i = 1; i < color.length; i++) {
+      color[i] = LETTERS[(int) Math.floor(Math.random() * 16)].charAt(0);
+    }
+    return new String(color);
+  }
+  
+}

neo4j/src/main/resources/interpreter-setting.json

@@ -0,0 +1,42 @@
+[
+  {
+    "group": "neo4j",
+    "name": "neo4j",
+    "className": "org.apache.zeppelin.graph.neo4j.Neo4jCypherInterpreter",
+    "properties": {
+      "neo4j.url": {
+        "envName": null,
+        "propertyName": "neo4j.url",
+        "defaultValue": "bolt://localhost:7687",
+        "description": "The Neo4j's BOLT url."
+      },
+      "neo4j.auth.type": {
+        "envName": null,
+        "propertyName": "neo4j.auth.type",
+        "defaultValue": "BASIC",
+        "description": "The Neo4j's authentication type (NONE, BASIC)."
+      },
+      "neo4j.auth.user": {
+        "envName": null,
+        "propertyName": "neo4j.auth.user",
+        "defaultValue": "",
+        "description": "The Neo4j user name."
+      },
+      "neo4j.auth.password": {
+        "envName": null,
+        "propertyName": "neo4j.auth.password",
+        "defaultValue": "",
+        "description": "The Neo4j user password."
+      },
+      "neo4j.max.concurrency": {
+        "envName": null,
+        "propertyName": "neo4j.max.concurrency",
+        "defaultValue": "50",
+        "description": "Max concurrency call from Zeppelin to Neo4j server."
+      }
+    },
+    "editor": {
+      "editOnDblClick": false
+    }
+  }
+]

neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java

@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.graph.neo4j;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.graph.GraphResult;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.neo4j.harness.ServerControls;
+import org.neo4j.harness.TestServerBuilders;
+
+import com.google.gson.Gson;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class Neo4jCypherInterpreterTest {
+
+  private Neo4jCypherInterpreter interpreter;
+
+  private InterpreterContext context;
+
+  private static ServerControls server;
+
+  private static final Gson gson = new Gson();
+
+  private static final String LABEL_PERSON = "Person";
+  private static final String REL_KNOWS = "KNOWS";
+
+  private static final String CYPHER_FOREACH = "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))";
+  private static final String CHPHER_UNWIND = "UNWIND range(1,1000) as x "
+        + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) "
+        + "CREATE (n)-[:%s]->(m)";
+
+  @BeforeClass
+  public static void setUpNeo4jServer() throws Exception {
+    server = TestServerBuilders.newInProcessBuilder()
+                .withConfig("dbms.security.auth_enabled","false")
+                .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10"))
+                .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS))
+                .newServer();
+  }
+
+  @AfterClass
+  public static void tearDownNeo4jServer() throws Exception {
+    server.close();
+  }
+  
+  @Before
+  public void setUpZeppelin() {
+    Properties p = new Properties();
+    p.setProperty(Neo4jConnectionManager.NEO4J_SERVER_URL, server.boltURI().toString());
+    p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString());
+    p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50");
+    interpreter = new Neo4jCypherInterpreter(p);
+    context = new InterpreterContext("note", "id", null, "title", "text",
+            new AuthenticationInfo(),
+            new HashMap<String, Object>(),
+            new GUI(),
+            new AngularObjectRegistry(new InterpreterGroup().getId(), null),
+            new LocalResourcePool("id"),
+            new LinkedList<InterpreterContextRunner>(),
+            new InterpreterOutput(null));
+  }
+
+  @After
+  public void tearDownZeppelin() throws Exception {
+    interpreter.close();
+  }
+
+  @Test
+  public void testTableWithArray() {
+    interpreter.open();
+    InterpreterResult result = interpreter.interpret("return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context);
+    assertEquals(Code.SUCCESS, result.code());
+    final String tableResult = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[1,2,3]\n";
+    assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
+    
+    result = interpreter.interpret("return 'a' as colA, 'b' as colB, [{key: \"value\"}, {key: 1}] as colC", context);
+    assertEquals(Code.SUCCESS, result.code());
+    final String tableResultWithMap = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n";
+    assertEquals(tableResultWithMap, result.toString().replace("%table ", StringUtils.EMPTY));
+  }
+
+  @Test
+  public void testCreateIndex() {
+    interpreter.open();
+    InterpreterResult result = interpreter.interpret("CREATE INDEX ON :Person(name)", context);
+    assertEquals(Code.SUCCESS, result.code());
+    assertEquals(StringUtils.EMPTY, result.toString());
+  }
+
+  @Test
+  public void testRenderTable() {
+    interpreter.open();
+    InterpreterResult result = interpreter.interpret("MATCH (n:Person) "
+    		+ "WHERE n.name IN ['name1', 'name2', 'name3'] "
+    		+ "RETURN n.name AS name, n.age AS age", context);
+    assertEquals(Code.SUCCESS, result.code());
+    final String tableResult = "name\tage\n\"name1\"\t1\n\"name2\"\t2\n\"name3\"\t3\n";
+    assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
+  }
+
+  @Test
+  public void testRenderMap() {
+    interpreter.open();
+    final String jsonQuery = "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object";
+    final String objectKey = "object.key";
+    final String objectListKey = "object.listKey";
+    InterpreterResult result = interpreter.interpret(jsonQuery, context);
+    assertEquals(Code.SUCCESS, result.code());
+    String[] rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
+    assertEquals(rows.length, 2);
+    List<String> header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
+    assertEquals(header.contains(objectKey), true);
+    assertEquals(header.contains(objectListKey), true);
+    List<String> row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value");
+    assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+
+    final String query = "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]},"
+    		+ "{key: \"value2\", listKey: [{inner: \"Map12\"}, {inner: \"Map22\"}]}] "
+    		+ "AS array UNWIND array AS object RETURN object";
+    result = interpreter.interpret(query, context);
+    assertEquals(Code.SUCCESS, result.code());
+    rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
+    assertEquals(rows.length, 3);
+    header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
+    assertEquals(header.contains(objectKey), true);
+    assertEquals(header.contains(objectListKey), true);
+    row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value");
+    assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+    row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value2");
+    assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]");
+
+    final String jsonListWithNullQuery = "WITH [{key: \"value\", listKey: null},"
+    		+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+    		+ "AS array UNWIND array AS object RETURN object";
+    result = interpreter.interpret(jsonListWithNullQuery, context);
+    assertEquals(Code.SUCCESS, result.code());
+    rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
+    assertEquals(rows.length, 3);
+    header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(header.contains(objectKey), true);
+    assertEquals(header.contains(objectListKey), true);
+    row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value");
+    assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY);
+    assertEquals(row.get(header.indexOf(objectListKey)), "");
+    row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value2");
+    assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+    
+    final String jsonListWithoutListKeyQuery = "WITH [{key: \"value\"},"
+    		+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+    		+ "AS array UNWIND array AS object RETURN object";
+    result = interpreter.interpret(jsonListWithoutListKeyQuery, context);
+    assertEquals(Code.SUCCESS, result.code());
+    rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
+    assertEquals(rows.length, 3);
+    header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(header.contains(objectKey), true);
+    assertEquals(header.contains(objectListKey), true);
+    row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value");
+    assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY);
+    row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
+    assertEquals(row.size(), header.size());
+    assertEquals(row.get(header.indexOf(objectKey)), "value2");
+    assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
+  }
+
+  @Test
+  public void testRenderNetwork() {
+    interpreter.open();
+    InterpreterResult result = interpreter.interpret("MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context);
+    GraphResult.Graph graph = gson.fromJson(result.toString().replace("%network ", StringUtils.EMPTY), GraphResult.Graph.class);
+    assertEquals(2, graph.getNodes().size());
+    assertEquals(true, graph.getNodes().iterator().next().getLabel().equals(LABEL_PERSON));
+    assertEquals(1, graph.getEdges().size());
+    assertEquals(true, graph.getEdges().iterator().next().getLabel().equals(REL_KNOWS));
+    assertEquals(1, graph.getLabels().size());
+    assertEquals(1, graph.getTypes().size());
+    assertEquals(true, graph.getLabels().containsKey(LABEL_PERSON));
+    assertEquals(REL_KNOWS, graph.getTypes().iterator().next());
+    assertEquals(Code.SUCCESS, result.code());
+  }
+
+  @Test
+  public void testFallingQuery() {
+    interpreter.open();
+    final String ERROR_MSG_EMPTY = "";
+    InterpreterResult result = interpreter.interpret(StringUtils.EMPTY, context);
+    assertEquals(Code.SUCCESS, result.code());
+    assertEquals(ERROR_MSG_EMPTY, result.toString());
+
+    result = interpreter.interpret(null, context);
+    assertEquals(Code.SUCCESS, result.code());
+    assertEquals(ERROR_MSG_EMPTY, result.toString());
+
+    result = interpreter.interpret("MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", context);
+    assertEquals(Code.ERROR, result.code());
+  }
+  
+}

pig/pom.xml

@@ -22,9 +22,10 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <artifactId>zeppelin</artifactId>
+        <artifactId>interpreter-parent</artifactId>
         <groupId>org.apache.zeppelin</groupId>
         <version>0.8.0-SNAPSHOT</version>
+        <relativePath>../interpreter-parent</relativePath>
     </parent>
 
     <groupId>org.apache.zeppelin</groupId>
@@ -36,6 +37,7 @@
     <url>http://zeppelin.apache.org</url>
 
     <properties>
+        <interpreter.name>pig</interpreter.name>
         <pig.version>0.17.0</pig.version>
         <hadoop.version>2.6.0</hadoop.version>
         <tez.version>0.7.0</tez.version>
@@ -175,58 +177,13 @@
         <plugins>
             <plugin>
                 <artifactId>maven-enforcer-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>enforce</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
             </plugin>
-
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>copy-dependencies</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy-dependencies</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/pig
-                            </outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>copy-artifact</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/../../interpreter/pig
-                            </outputDirectory>
-                            <overWriteReleases>false</overWriteReleases>
-                            <overWriteSnapshots>false</overWriteSnapshots>
-                            <overWriteIfNewer>true</overWriteIfNewer>
-                            <includeScope>runtime</includeScope>
-                            <artifactItems>
-                                <artifactItem>
-                                    <groupId>${project.groupId}</groupId>
-                                    <artifactId>${project.artifactId}</artifactId>
-                                    <version>${project.version}</version>
-                                    <type>${project.packaging}</type>
-                                </artifactItem>
-                            </artifactItems>
-                        </configuration>
-                    </execution>
-                </executions>
             </plugin>
-
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
             <plugin>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>

pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java

@@ -18,7 +18,6 @@
 package org.apache.zeppelin.pig;
 
 import org.apache.commons.io.output.ByteArrayOutputStream;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.pig.PigServer;
 import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -60,7 +59,7 @@ public class PigInterpreter extends BasePigInterpreter {
     }
     try {
       pigServer = new PigServer(execType);
-      for (Map.Entry entry : getProperty().entrySet()) {
+      for (Map.Entry entry : getProperties().entrySet()) {
         if (!entry.getKey().toString().startsWith("zeppelin.")) {
           pigServer.getPigContext().getProperties().setProperty(entry.getKey().toString(),
               entry.getValue().toString());

pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java

@@ -55,7 +55,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
   }
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
     pigServer = getPigInterpreter().getPigServer();
     maxResult = Integer.parseInt(getProperty(MAX_RESULTS));
   }
@@ -159,7 +159,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
     return this.pigServer;
   }
 
-  private PigInterpreter getPigInterpreter() {
+  private PigInterpreter getPigInterpreter() throws InterpreterException {
     LazyOpenInterpreter lazy = null;
     PigInterpreter pig = null;
     Interpreter p = getInterpreterInTheSameSessionByClassName(PigInterpreter.class.getName());

pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java

@@ -21,6 +21,7 @@ package org.apache.zeppelin.pig;
 import org.apache.commons.io.IOUtils;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.junit.After;
@@ -48,7 +49,7 @@ public class PigQueryInterpreterTest {
   private InterpreterContext context;
 
   @Before
-  public void setUp() {
+  public void setUp() throws InterpreterException {
     Properties properties = new Properties();
     properties.put("zeppelin.pig.execType", "local");
     properties.put("zeppelin.pig.maxResult", "20");

pom.xml

@@ -76,6 +76,7 @@
     <module>bigquery</module>
     <module>alluxio</module>
     <module>scio</module>
+    <module>neo4j</module>
     <module>zeppelin-web</module>
     <module>zeppelin-server</module>
     <module>zeppelin-jupyter</module>
@@ -93,7 +94,7 @@
     <node.version>v6.9.1</node.version>
     <yarn.version>v0.18.1</yarn.version>
     <npm.version>4.2.0</npm.version>
-    <plugin.frontned.version>1.3</plugin.frontned.version>
+    <plugin.frontend.version>1.3</plugin.frontend.version>
 
     <!-- common library versions -->
     <slf4j.version>1.7.10</slf4j.version>
@@ -101,7 +102,6 @@
     <libthrift.version>0.9.2</libthrift.version>
     <gson.version>2.2</gson.version>
     <gson-extras.version>0.2.1</gson-extras.version>
-    <guava.version>15.0</guava.version>
     <jetty.version>9.2.15.v20160210</jetty.version>
     <httpcomponents.core.version>4.4.1</httpcomponents.core.version>
     <httpcomponents.client.version>4.5.1</httpcomponents.client.version>
@@ -246,12 +246,6 @@
         <version>${commons.cli.version}</version>
       </dependency>
 
-      <dependency>
-        <groupId>com.google.guava</groupId>
-        <artifactId>guava</artifactId>
-        <version>${guava.version}</version>
-      </dependency>
-
       <!-- Apache Shiro -->
       <dependency>
         <groupId>org.apache.shiro</groupId>
@@ -404,7 +398,7 @@
             </goals>
             <configuration>
               <failOnViolation>true</failOnViolation>
-              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*</excludes>
+              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
             </configuration>
           </execution>
           <execution>
@@ -414,7 +408,7 @@
               <goal>checkstyle-aggregate</goal>
             </goals>
             <configuration>
-              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*</excludes>
+              <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes>
             </configuration>
           </execution>
         </executions>
@@ -1082,6 +1076,8 @@
                <!--The following files are mechanical-->
               <exclude>**/R/rzeppelin/DESCRIPTION</exclude>
               <exclude>**/R/rzeppelin/NAMESPACE</exclude>
+
+              <exclude>python/src/main/resources/grpc/**/*</exclude>
             </excludes>
           </configuration>
 

python/README.md

@@ -50,3 +50,22 @@ mvn -Dpython.test.exclude='' test -pl python -am
  * Matplotlib figures are displayed inline with the notebook automatically using a built-in backend for zeppelin in conjunction with a post-execute hook.
 
  * `%python.sql` support for Pandas DataFrames is optional and provided using https://github.com/yhat/pandasql if user have one installed
+
+
+# IPython Overview
+IPython interpreter for Apache Zeppelin
+
+# IPython Requirements
+You need to install the following python packages to make the IPython interpreter work.
+ * jupyter 5.x
+ * IPython
+ * ipykernel
+ * grpcio
+ 
+If you have installed anaconda, then you just need to install grpc.
+
+# IPython Architecture
+Current interpreter delegate the whole work to ipython kernel via `jupyter_client`. Zeppelin would launch a python process which host the ipython kernel.
+Zeppelin interpreter process will communicate with the python process via `grpc`. Ideally every feature works in IPython should work in Zeppelin as well.
+
+

python/pom.xml

@@ -20,10 +20,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <artifactId>zeppelin</artifactId>
+    <artifactId>interpreter-parent</artifactId>
     <groupId>org.apache.zeppelin</groupId>
     <version>0.8.0-SNAPSHOT</version>
-    <relativePath>..</relativePath>
+    <relativePath>../interpreter-parent</relativePath>
   </parent>
 
   <groupId>org.apache.zeppelin</groupId>
@@ -33,12 +33,16 @@
   <name>Zeppelin: Python interpreter</name>
 
   <properties>
+    <interpreter.name>python</interpreter.name>
     <python.py4j.version>0.9.2</python.py4j.version>
     <python.test.exclude>
         **/PythonInterpreterWithPythonInstalledTest.java,
         **/PythonInterpreterPandasSqlTest.java,
         **/PythonInterpreterMatplotlibTest.java
     </python.test.exclude>
+    <pypi.repo.url>https://pypi.python.org/packages</pypi.repo.url>
+    <python.py4j.repo.folder>/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</python.py4j.repo.folder>
+    <grpc.version>1.4.0</grpc.version>
   </properties>
 
   <dependencies>
@@ -71,6 +75,28 @@
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-protobuf</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-stub</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>18.0</version>
+    </dependency>
+
     <!-- test libraries -->
     <dependency>
       <groupId>junit</groupId>
@@ -86,14 +112,32 @@
   </dependencies>
 
   <build>
+
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.4.1.Final</version>
+      </extension>
+    </extensions>
+
     <plugins>
+
       <plugin>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <version>1.3.1</version>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.5.0</version>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}</protocArtifact>
+          <pluginId>grpc-java</pluginId>
+          <pluginArtifact>io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}</pluginArtifact>
+        </configuration>
         <executions>
           <execution>
-            <id>enforce</id>
-            <phase>none</phase>
+            <goals>
+              <goal>compile</goal>
+              <goal>compile-custom</goal>
+            </goals>
           </execution>
         </executions>
       </plugin>
@@ -107,7 +151,7 @@
             <phase>package</phase>
             <goals><goal>download-single</goal></goals>
             <configuration>
-              <url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
+              <url>${pypi.repo.url}${python.py4j.repo.folder}</url>
               <fromFile>py4j-${python.py4j.version}.zip</fromFile>
               <toFile>${project.build.directory}/../../interpreter/python/py4j-${python.py4j.version}.zip</toFile>
             </configuration>
@@ -134,6 +178,19 @@
         </executions>
       </plugin>
 
+      <!-- publish test jar as well so that spark module can use it -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>3.0.2</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
 
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -145,47 +202,14 @@
         </configuration>
       </plugin>
 
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+      </plugin>
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>2.8</version>
-        <executions>
-          <execution>
-            <id>copy-dependencies</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/python</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-artifact</id>
-            <phase>package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>${project.build.directory}/../../interpreter/python</outputDirectory>
-              <overWriteReleases>false</overWriteReleases>
-              <overWriteSnapshots>false</overWriteSnapshots>
-              <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>${project.groupId}</groupId>
-                  <artifactId>${project.artifactId}</artifactId>
-                  <version>${project.version}</version>
-                  <type>${project.packaging}</type>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
-        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-resources-plugin</artifactId>
       </plugin>
 
     </plugins>

python/src/main/java/org/apache/zeppelin/python/IPythonClient.java

@@ -0,0 +1,211 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.stub.StreamObserver;
+import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
+import org.apache.zeppelin.python.proto.CancelRequest;
+import org.apache.zeppelin.python.proto.CancelResponse;
+import org.apache.zeppelin.python.proto.CompletionRequest;
+import org.apache.zeppelin.python.proto.CompletionResponse;
+import org.apache.zeppelin.python.proto.ExecuteRequest;
+import org.apache.zeppelin.python.proto.ExecuteResponse;
+import org.apache.zeppelin.python.proto.ExecuteStatus;
+import org.apache.zeppelin.python.proto.IPythonGrpc;
+import org.apache.zeppelin.python.proto.OutputType;
+import org.apache.zeppelin.python.proto.StatusRequest;
+import org.apache.zeppelin.python.proto.StatusResponse;
+import org.apache.zeppelin.python.proto.StopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.security.SecureRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Grpc client for IPython kernel
+ */
+public class IPythonClient {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonClient.class.getName());
+
+  private final ManagedChannel channel;
+  private final IPythonGrpc.IPythonBlockingStub blockingStub;
+  private final IPythonGrpc.IPythonStub asyncStub;
+
+  private SecureRandom random = new SecureRandom();
+
+  /**
+   * Construct client for accessing RouteGuide server at {@code host:port}.
+   */
+  public IPythonClient(String host, int port) {
+    this(ManagedChannelBuilder.forAddress(host, port).usePlaintext(true));
+  }
+
+  /**
+   * Construct client for accessing RouteGuide server using the existing channel.
+   */
+  public IPythonClient(ManagedChannelBuilder<?> channelBuilder) {
+    channel = channelBuilder.build();
+    blockingStub = IPythonGrpc.newBlockingStub(channel);
+    asyncStub = IPythonGrpc.newStub(channel);
+  }
+
+  public void shutdown() throws InterruptedException {
+    channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
+  }
+
+  // execute the code and make the output as streaming by writing it to InterpreterOutputStream
+  // one by one.
+  public ExecuteResponse stream_execute(ExecuteRequest request,
+                                        final InterpreterOutputStream interpreterOutput) {
+    final ExecuteResponse.Builder finalResponseBuilder = ExecuteResponse.newBuilder()
+        .setStatus(ExecuteStatus.SUCCESS);
+    final AtomicBoolean completedFlag = new AtomicBoolean(false);
+    LOGGER.debug("stream_execute code:\n" + request.getCode());
+    asyncStub.execute(request, new StreamObserver<ExecuteResponse>() {
+      int index = 0;
+      boolean isPreviousOutputImage = false;
+
+      @Override
+      public void onNext(ExecuteResponse executeResponse) {
+        if (executeResponse.getType() == OutputType.TEXT) {
+          try {
+            LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput());
+            if (isPreviousOutputImage) {
+              // add '\n' when switch from image to text
+              interpreterOutput.write("\n%text ".getBytes());
+            }
+            isPreviousOutputImage = false;
+            interpreterOutput.write(executeResponse.getOutput().getBytes());
+            interpreterOutput.getInterpreterOutput().flush();
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+        }
+        if (executeResponse.getType() == OutputType.IMAGE) {
+          try {
+            LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA");
+            if (index != 0) {
+              // add '\n' if this is the not the first element. otherwise it would mix the image
+              // with the text
+              interpreterOutput.write("\n".getBytes());
+            }
+            interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes());
+            interpreterOutput.getInterpreterOutput().flush();
+            isPreviousOutputImage = true;
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+        }
+        if (executeResponse.getStatus() == ExecuteStatus.ERROR) {
+          // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse would
+          // be SUCCESS.
+          finalResponseBuilder.setStatus(ExecuteStatus.ERROR);
+        }
+        index++;
+      }
+
+      @Override
+      public void onError(Throwable throwable) {
+        try {
+          interpreterOutput.getInterpreterOutput().flush();
+        } catch (IOException e) {
+          LOGGER.error("Unexpected IOException", e);
+        }
+        LOGGER.error("Fail to call IPython grpc", throwable);
+      }
+
+      @Override
+      public void onCompleted() {
+        synchronized (completedFlag) {
+          try {
+            LOGGER.debug("stream_execute is completed");
+            interpreterOutput.getInterpreterOutput().flush();
+          } catch (IOException e) {
+            LOGGER.error("Unexpected IOException", e);
+          }
+          completedFlag.set(true);
+          completedFlag.notify();
+        }
+      }
+    });
+
+    synchronized (completedFlag) {
+      if (!completedFlag.get()) {
+        try {
+          completedFlag.wait();
+        } catch (InterruptedException e) {
+          LOGGER.error("Unexpected Interruption", e);
+        }
+      }
+    }
+    return finalResponseBuilder.build();
+  }
+
+  // blocking execute the code
+  public ExecuteResponse block_execute(ExecuteRequest request) {
+    ExecuteResponse.Builder responseBuilder = ExecuteResponse.newBuilder();
+    responseBuilder.setStatus(ExecuteStatus.SUCCESS);
+    Iterator<ExecuteResponse> iter = blockingStub.execute(request);
+    StringBuilder outputBuilder = new StringBuilder();
+    while (iter.hasNext()) {
+      ExecuteResponse nextResponse = iter.next();
+      if (nextResponse.getStatus() == ExecuteStatus.ERROR) {
+        responseBuilder.setStatus(ExecuteStatus.ERROR);
+      }
+      outputBuilder.append(nextResponse.getOutput());
+    }
+    responseBuilder.setOutput(outputBuilder.toString());
+    return responseBuilder.build();
+  }
+
+  public CancelResponse cancel(CancelRequest request) {
+    return blockingStub.cancel(request);
+  }
+
+  public CompletionResponse complete(CompletionRequest request) {
+    return blockingStub.complete(request);
+  }
+
+  public StatusResponse status(StatusRequest request) {
+    return blockingStub.status(request);
+  }
+
+  public void stop(StopRequest request) {
+    asyncStub.stop(request, null);
+  }
+
+
+  public static void main(String[] args) {
+    IPythonClient client = new IPythonClient("localhost", 50053);
+    client.status(StatusRequest.newBuilder().build());
+
+    ExecuteResponse response = client.block_execute(ExecuteRequest.newBuilder().
+        setCode("abcd=2").build());
+    System.out.println(response.getOutput());
+
+  }
+}

python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java

@@ -0,0 +1,377 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.exec.ExecuteResultHandler;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.LogOutputStream;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
+import org.apache.zeppelin.python.proto.CancelRequest;
+import org.apache.zeppelin.python.proto.CompletionRequest;
+import org.apache.zeppelin.python.proto.CompletionResponse;
+import org.apache.zeppelin.python.proto.ExecuteRequest;
+import org.apache.zeppelin.python.proto.ExecuteResponse;
+import org.apache.zeppelin.python.proto.ExecuteStatus;
+import org.apache.zeppelin.python.proto.IPythonStatus;
+import org.apache.zeppelin.python.proto.StatusRequest;
+import org.apache.zeppelin.python.proto.StatusResponse;
+import org.apache.zeppelin.python.proto.StopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import py4j.GatewayServer;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * IPython Interpreter for Zeppelin
+ */
+public class IPythonInterpreter extends Interpreter implements ExecuteResultHandler {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreter.class);
+
+  private ExecuteWatchdog watchDog;
+  private IPythonClient ipythonClient;
+  private GatewayServer gatewayServer;
+
+  private PythonZeppelinContext zeppelinContext;
+  private String pythonExecutable;
+  private long ipythonLaunchTimeout;
+  private String additionalPythonPath;
+  private String additionalPythonInitFile;
+  private boolean useBuiltinPy4j = true;
+
+  private InterpreterOutputStream interpreterOutput = new InterpreterOutputStream(LOGGER);
+
+  public IPythonInterpreter(Properties properties) {
+    super(properties);
+  }
+
+  /**
+   * Sub class can customize the interpreter by adding more python packages under PYTHONPATH.
+   * e.g. PySparkInterpreter
+   *
+   * @param additionalPythonPath
+   */
+  public void setAdditionalPythonPath(String additionalPythonPath) {
+    LOGGER.info("setAdditionalPythonPath: " + additionalPythonPath);
+    this.additionalPythonPath = additionalPythonPath;
+  }
+
+  /**
+   * Sub class can customize the interpreter by running additional python init code.
+   * e.g. PySparkInterpreter
+   *
+   * @param additionalPythonInitFile
+   */
+  public void setAdditionalPythonInitFile(String additionalPythonInitFile) {
+    this.additionalPythonInitFile = additionalPythonInitFile;
+  }
+
+  public void setAddBulitinPy4j(boolean add) {
+    this.useBuiltinPy4j = add;
+  }
+
+  @Override
+  public void open() throws InterpreterException {
+    try {
+      if (ipythonClient != null) {
+        // IPythonInterpreter might already been opened by PythonInterpreter
+        return;
+      }
+      pythonExecutable = getProperty("zeppelin.python", "python");
+      LOGGER.info("Python Exec: " + pythonExecutable);
+
+      ipythonLaunchTimeout = Long.parseLong(
+          getProperty("zeppelin.ipython.launch.timeout", "30000"));
+      this.zeppelinContext = new PythonZeppelinContext(
+          getInterpreterGroup().getInterpreterHookRegistry(),
+          Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000")));
+      int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+      int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
+      LOGGER.info("Launching IPython Kernel at port: " + ipythonPort);
+      LOGGER.info("Launching JVM Gateway at port: " + jvmGatewayPort);
+      ipythonClient = new IPythonClient("127.0.0.1", ipythonPort);
+      launchIPythonKernel(ipythonPort);
+      setupJVMGateway(jvmGatewayPort);
+    } catch (Exception e) {
+      throw new RuntimeException("Fail to open IPythonInterpreter", e);
+    }
+  }
+
+  public boolean checkIPythonPrerequisite() {
+    ProcessBuilder processBuilder = new ProcessBuilder("pip", "freeze");
+    try {
+      File stderrFile = File.createTempFile("zeppelin", ".txt");
+      processBuilder.redirectError(stderrFile);
+      File stdoutFile = File.createTempFile("zeppelin", ".txt");
+      processBuilder.redirectOutput(stdoutFile);
+
+      Process proc = processBuilder.start();
+      int ret = proc.waitFor();
+      if (ret != 0) {
+        LOGGER.warn("Fail to run pip freeze.\n" +
+            IOUtils.toString(new FileInputStream(stderrFile)));
+        return false;
+      }
+      String freezeOutput = IOUtils.toString(new FileInputStream(stdoutFile));
+      if (!freezeOutput.contains("jupyter-client=")) {
+        InterpreterContext.get().out.write("jupyter-client is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("ipykernel=")) {
+        InterpreterContext.get().out.write("ipkernel is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("ipython=")) {
+        InterpreterContext.get().out.write("ipython is not installed\n".getBytes());
+        return false;
+      }
+      if (!freezeOutput.contains("grpcio=")) {
+        InterpreterContext.get().out.write("grpcio is not installed\n".getBytes());
+        return false;
+      }
+      LOGGER.info("IPython prerequisite is meet");
+      return true;
+    } catch (Exception e) {
+      LOGGER.warn("Fail to checkIPythonPrerequisite", e);
+      return false;
+    }
+  }
+
+  private void setupJVMGateway(int jvmGatewayPort) throws IOException {
+    gatewayServer = new GatewayServer(this, jvmGatewayPort);
+    gatewayServer.start();
+
+    InputStream input =
+        getClass().getClassLoader().getResourceAsStream("grpc/python/zeppelin_python.py");
+    List<String> lines = IOUtils.readLines(input);
+    ExecuteResponse response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+        .setCode(StringUtils.join(lines, System.lineSeparator())
+            .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")).build());
+    if (response.getStatus() == ExecuteStatus.ERROR) {
+      throw new IOException("Fail to setup JVMGateway\n" + response.getOutput());
+    }
+
+    if (additionalPythonInitFile != null) {
+      input = getClass().getClassLoader().getResourceAsStream(additionalPythonInitFile);
+      lines = IOUtils.readLines(input);
+      response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+          .setCode(StringUtils.join(lines, System.lineSeparator())
+              .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "")).build());
+      if (response.getStatus() == ExecuteStatus.ERROR) {
+        throw new IOException("Fail to run additional Python init file: "
+            + additionalPythonInitFile + "\n" + response.getOutput());
+      }
+    }
+  }
+
+
+  private void launchIPythonKernel(int ipythonPort)
+      throws IOException, URISyntaxException {
+    // copy the python scripts to a temp directory, then launch ipython kernel in that folder
+    File tmpPythonScriptFolder = Files.createTempDirectory("zeppelin_ipython").toFile();
+    String[] ipythonScripts = {"ipython_server.py", "ipython_pb2.py", "ipython_pb2_grpc.py"};
+    for (String ipythonScript : ipythonScripts) {
+      URL url = getClass().getClassLoader().getResource("grpc/python"
+          + "/" + ipythonScript);
+      FileUtils.copyURLToFile(url, new File(tmpPythonScriptFolder, ipythonScript));
+    }
+
+    CommandLine cmd = CommandLine.parse(pythonExecutable);
+    cmd.addArgument(tmpPythonScriptFolder.getAbsolutePath() + "/ipython_server.py");
+    cmd.addArgument(ipythonPort + "");
+    DefaultExecutor executor = new DefaultExecutor();
+    ProcessLogOutputStream processOutput = new ProcessLogOutputStream(LOGGER);
+    executor.setStreamHandler(new PumpStreamHandler(processOutput));
+    watchDog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
+    executor.setWatchdog(watchDog);
+
+    if (useBuiltinPy4j) {
+      String py4jLibPath = null;
+      if (System.getenv("ZEPPELIN_HOME") != null) {
+        py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator
+            + PythonInterpreter.ZEPPELIN_PY4JPATH;
+      } else {
+        Path workingPath = Paths.get("..").toAbsolutePath();
+        py4jLibPath = workingPath + File.separator + PythonInterpreter.ZEPPELIN_PY4JPATH;
+      }
+      if (additionalPythonPath != null) {
+        // put the py4j at the end, because additionalPythonPath may already contain py4j.
+        // e.g. PySparkInterpreter
+        additionalPythonPath = additionalPythonPath + ":" + py4jLibPath;
+      } else {
+        additionalPythonPath = py4jLibPath;
+      }
+    }
+
+    Map<String, String> envs = setupIPythonEnv();
+    executor.execute(cmd, envs, this);
+
+    // wait until IPython kernel is started or timeout
+    long startTime = System.currentTimeMillis();
+    while (true) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+        LOGGER.error("Interrupted by something", e);
+      }
+
+      try {
+        StatusResponse response = ipythonClient.status(StatusRequest.newBuilder().build());
+        if (response.getStatus() == IPythonStatus.RUNNING) {
+          LOGGER.info("IPython Kernel is Running");
+          break;
+        } else {
+          LOGGER.info("Wait for IPython Kernel to be started");
+        }
+      } catch (Exception e) {
+        // ignore the exception, because is may happen when grpc server has not started yet.
+        LOGGER.info("Wait for IPython Kernel to be started");
+      }
+
+      if ((System.currentTimeMillis() - startTime) > ipythonLaunchTimeout) {
+        throw new IOException("Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000
+            + " seconds");
+      }
+    }
+  }
+
+  protected Map<String, String> setupIPythonEnv() throws IOException {
+    Map<String, String> envs = EnvironmentUtils.getProcEnvironment();
+    if (envs.containsKey("PYTHONPATH")) {
+      if (additionalPythonPath != null) {
+        envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH"));
+      }
+    } else {
+      envs.put("PYTHONPATH", additionalPythonPath);
+    }
+    LOGGER.info("PYTHONPATH:" + envs.get("PYTHONPATH"));
+    return envs;
+  }
+
+  @Override
+  public void close() {
+    if (watchDog != null) {
+      LOGGER.debug("Kill IPython Process");
+      ipythonClient.stop(StopRequest.newBuilder().build());
+      watchDog.destroyProcess();
+      gatewayServer.shutdown();
+    }
+  }
+
+  @Override
+  public InterpreterResult interpret(String st, InterpreterContext context) {
+    zeppelinContext.setGui(context.getGui());
+    interpreterOutput.setInterpreterOutput(context.out);
+    ExecuteResponse response =
+        ipythonClient.stream_execute(ExecuteRequest.newBuilder().setCode(st).build(),
+            interpreterOutput);
+    try {
+      interpreterOutput.getInterpreterOutput().flush();
+    } catch (IOException e) {
+      throw new RuntimeException("Fail to write output", e);
+    }
+    InterpreterResult result = new InterpreterResult(
+        InterpreterResult.Code.valueOf(response.getStatus().name()));
+    return result;
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+    ipythonClient.cancel(CancelRequest.newBuilder().build());
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.SIMPLE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return 0;
+  }
+
+  @Override
+  public List<InterpreterCompletion> completion(String buf, int cursor,
+                                                InterpreterContext interpreterContext) {
+    List<InterpreterCompletion> completions = new ArrayList<>();
+    CompletionResponse response =
+        ipythonClient.complete(
+            CompletionRequest.getDefaultInstance().newBuilder().setCode(buf)
+                .setCursor(cursor).build());
+    for (int i = 0; i < response.getMatchesCount(); i++) {
+      completions.add(new InterpreterCompletion(
+          response.getMatches(i), response.getMatches(i), ""));
+    }
+    return completions;
+  }
+
+  public PythonZeppelinContext getZeppelinContext() {
+    return zeppelinContext;
+  }
+
+  @Override
+  public void onProcessComplete(int exitValue) {
+    LOGGER.warn("Python Process is completed with exitValue: " + exitValue);
+  }
+
+  @Override
+  public void onProcessFailed(ExecuteException e) {
+    LOGGER.warn("Exception happens in Python Process", e);
+  }
+
+  private static class ProcessLogOutputStream extends LogOutputStream {
+
+    private Logger logger;
+
+    public ProcessLogOutputStream(Logger logger) {
+      this.logger = logger;
+    }
+
+    @Override
+    protected void processLine(String s, int i) {
+      this.logger.debug("Process Output: " + s);
+    }
+  }
+}

python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java

@@ -50,6 +50,8 @@ public class PythonCondaInterpreter extends Interpreter {
   public static final Pattern PATTERN_COMMAND_HELP = Pattern.compile("help");
   public static final Pattern PATTERN_COMMAND_INFO = Pattern.compile("info");
 
+  private String currentCondaEnvName = StringUtils.EMPTY;
+
   public PythonCondaInterpreter(Properties property) {
     super(property);
   }
@@ -65,7 +67,8 @@ public class PythonCondaInterpreter extends Interpreter {
   }
 
   @Override
-  public InterpreterResult interpret(String st, InterpreterContext context) {
+  public InterpreterResult interpret(String st, InterpreterContext context)
+      throws InterpreterException {
     InterpreterOutput out = context.out;
     Matcher activateMatcher = PATTERN_COMMAND_ACTIVATE.matcher(st);
     Matcher createMatcher = PATTERN_COMMAND_CREATE.matcher(st);
@@ -112,8 +115,19 @@ public class PythonCondaInterpreter extends Interpreter {
     }
   }
 
+  public String getCurrentCondaEnvName() {
+    return currentCondaEnvName;
+  }
+
+  public void setCurrentCondaEnvName(String currentCondaEnvName) {
+    if (currentCondaEnvName == null) {
+      currentCondaEnvName = StringUtils.EMPTY;
+    }
+    this.currentCondaEnvName = currentCondaEnvName;
+  }
+
   private void changePythonEnvironment(String envName)
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, InterpreterException {
     PythonInterpreter python = getPythonInterpreter();
     String binPath = null;
     if (envName == null) {
@@ -130,16 +144,17 @@ public class PythonCondaInterpreter extends Interpreter {
         }
       }
     }
+    setCurrentCondaEnvName(envName);
     python.setPythonCommand(binPath);
   }
 
-  private void restartPythonProcess() {
+  private void restartPythonProcess() throws InterpreterException {
     PythonInterpreter python = getPythonInterpreter();
     python.close();
     python.open();
   }
 
-  protected PythonInterpreter getPythonInterpreter() {
+  protected PythonInterpreter getPythonInterpreter() throws InterpreterException {
     LazyOpenInterpreter lazy = null;
     PythonInterpreter python = null;
     Interpreter p =
@@ -199,7 +214,7 @@ public class PythonCondaInterpreter extends Interpreter {
   }
 
   private InterpreterResult runCondaActivate(String envName)
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, InterpreterException {
 
     if (null == envName || envName.isEmpty()) {
       return new InterpreterResult(Code.ERROR, "Env name should be specified");
@@ -212,7 +227,7 @@ public class PythonCondaInterpreter extends Interpreter {
   }
 
   private InterpreterResult runCondaDeactivate()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, InterpreterException {
 
     changePythonEnvironment(null);
     restartPythonProcess();
@@ -221,8 +236,12 @@ public class PythonCondaInterpreter extends Interpreter {
 
   private String runCondaList() throws IOException, InterruptedException {
     List<String> commands = new ArrayList<String>();
-    commands.add("conda");
-    commands.add("list");
+    commands.add(0, "conda");
+    commands.add(1, "list");
+    if (!getCurrentCondaEnvName().isEmpty()) {
+      commands.add(2, "-n");
+      commands.add(3, getCurrentCondaEnvName());
+    }
 
     return runCondaCommandForTableOutput("Installed Package List", commands);
   }
@@ -259,6 +278,10 @@ public class PythonCondaInterpreter extends Interpreter {
     restArgs.add(0, "conda");
     restArgs.add(1, "install");
     restArgs.add(2, "--yes");
+    if (!getCurrentCondaEnvName().isEmpty()) {
+      restArgs.add(3, "-n");
+      restArgs.add(4, getCurrentCondaEnvName());
+    }
 
     return runCondaCommandForTextOutput("Package Installation", restArgs);
   }
@@ -269,6 +292,10 @@ public class PythonCondaInterpreter extends Interpreter {
     restArgs.add(0, "conda");
     restArgs.add(1, "uninstall");
     restArgs.add(2, "--yes");
+    if (!getCurrentCondaEnvName().isEmpty()) {
+      restArgs.add(3, "-n");
+      restArgs.add(4, getCurrentCondaEnvName());
+    }
 
     return runCondaCommandForTextOutput("Package Uninstallation", restArgs);
   }
@@ -349,10 +376,16 @@ public class PythonCondaInterpreter extends Interpreter {
    */
   @Override
   public Scheduler getScheduler() {
-    PythonInterpreter pythonInterpreter = getPythonInterpreter();
-    if (pythonInterpreter != null) {
-      return pythonInterpreter.getScheduler();
-    } else {
+    PythonInterpreter pythonInterpreter = null;
+    try {
+      pythonInterpreter = getPythonInterpreter();
+      if (pythonInterpreter != null) {
+        return pythonInterpreter.getScheduler();
+      } else {
+        return null;
+      }
+    } catch (InterpreterException e) {
+      e.printStackTrace();
       return null;
     }
   }

python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java

@@ -56,7 +56,8 @@ public class PythonDockerInterpreter extends Interpreter {
   }
 
   @Override
-  public InterpreterResult interpret(String st, InterpreterContext context) {
+  public InterpreterResult interpret(String st, InterpreterContext context)
+      throws InterpreterException {
     File pythonScript = new File(getPythonInterpreter().getScriptPath());
     InterpreterOutput out = context.out;
 
@@ -105,7 +106,7 @@ public class PythonDockerInterpreter extends Interpreter {
   }
 
 
-  public void setPythonCommand(String cmd) {
+  public void setPythonCommand(String cmd) throws InterpreterException {
     PythonInterpreter python = getPythonInterpreter();
     python.setPythonCommand(cmd);
   }
@@ -140,21 +141,27 @@ public class PythonDockerInterpreter extends Interpreter {
    */
   @Override
   public Scheduler getScheduler() {
-    PythonInterpreter pythonInterpreter = getPythonInterpreter();
-    if (pythonInterpreter != null) {
-      return pythonInterpreter.getScheduler();
-    } else {
+    PythonInterpreter pythonInterpreter = null;
+    try {
+      pythonInterpreter = getPythonInterpreter();
+      if (pythonInterpreter != null) {
+        return pythonInterpreter.getScheduler();
+      } else {
+        return null;
+      }
+    } catch (InterpreterException e) {
+      e.printStackTrace();
       return null;
     }
   }
 
-  private void restartPythonProcess() {
+  private void restartPythonProcess() throws InterpreterException {
     PythonInterpreter python = getPythonInterpreter();
     python.close();
     python.open();
   }
 
-  protected PythonInterpreter getPythonInterpreter() {
+  protected PythonInterpreter getPythonInterpreter() throws InterpreterException {
     LazyOpenInterpreter lazy = null;
     PythonInterpreter python = null;
     Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
@@ -173,7 +180,7 @@ public class PythonDockerInterpreter extends Interpreter {
     return python;
   }
 
-  public boolean pull(InterpreterOutput out, String image) {
+  public boolean pull(InterpreterOutput out, String image) throws InterpreterException {
     int exit = 0;
     try {
       exit = runCommand(out, "docker", "pull", image);

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -57,7 +57,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import py4j.GatewayServer;
-import py4j.commands.Command;
 
 /**
  * Python interpreter for Zeppelin.
@@ -91,6 +90,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   private static final int MAX_TIMEOUT_SEC = 10;
 
   private long pythonPid = 0;
+  private IPythonInterpreter iPythonInterpreter;
 
   Integer statementSetNotifier = new Integer(0);
 
@@ -100,7 +100,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
       File scriptFile = File.createTempFile("zeppelin_python-", ".py", new File("/tmp"));
       scriptPath = scriptFile.getAbsolutePath();
     } catch (IOException e) {
-      throw new InterpreterException(e);
+      throw new RuntimeException(e);
     }
   }
 
@@ -115,7 +115,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     return path;
   }
 
-  private void createPythonScript() {
+  private void createPythonScript() throws InterpreterException {
     File out = new File(scriptPath);
 
     if (out.exists() && out.isDirectory()) {
@@ -130,7 +130,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     return scriptPath;
   }
 
-  private void copyFile(File out, String sourceFile) {
+  private void copyFile(File out, String sourceFile) throws InterpreterException {
     ClassLoader classLoader = getClass().getClassLoader();
     try {
       FileOutputStream outStream = new FileOutputStream(out);
@@ -143,7 +143,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     }
   }
 
-  private void createGatewayServerAndStartScript() throws UnknownHostException {
+  private void createGatewayServerAndStartScript()
+      throws UnknownHostException, InterpreterException {
     createPythonScript();
     if (System.getenv("ZEPPELIN_HOME") != null) {
       py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
@@ -218,7 +219,38 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   }
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
+    // try IPythonInterpreter first. If it is not available, we will fallback to the original
+    // python interpreter implementation.
+    iPythonInterpreter = getIPythonInterpreter();
+    if (getProperty("zeppelin.python.useIPython", "true").equals("true") &&
+      iPythonInterpreter.checkIPythonPrerequisite()) {
+      try {
+        iPythonInterpreter.open();
+        if (InterpreterContext.get() != null) {
+          InterpreterContext.get().out.write(("IPython is available, " +
+              "use IPython for PythonInterpreter\n")
+              .getBytes());
+        }
+        LOG.info("Use IPythonInterpreter to replace PythonInterpreter");
+        return;
+      } catch (Exception e) {
+        iPythonInterpreter = null;
+      }
+    }
+    // reset iPythonInterpreter to null
+    iPythonInterpreter = null;
+
+    try {
+      if (InterpreterContext.get() != null) {
+        InterpreterContext.get().out.write(("IPython is not available, " +
+            "use the native PythonInterpreter\n")
+            .getBytes());
+      }
+    } catch (IOException e) {
+      LOG.warn("Fail to write InterpreterOutput", e.getMessage());
+    }
+
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
@@ -232,8 +264,27 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     }
   }
 
+  private IPythonInterpreter getIPythonInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    IPythonInterpreter ipython = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(IPythonInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    ipython = (IPythonInterpreter) p;
+    return ipython;
+  }
+
   @Override
   public void close() {
+    if (iPythonInterpreter != null) {
+      iPythonInterpreter.close();
+      return;
+    }
     pythonscriptRunning = false;
     pythonScriptInitialized = false;
 
@@ -318,7 +369,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   }
 
   @Override
-  public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
+  public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter)
+      throws InterpreterException {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.interpret(cmd, contextInterpreter);
+    }
     if (cmd == null || cmd.isEmpty()) {
       return new InterpreterResult(Code.SUCCESS, "");
     }
@@ -411,6 +466,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public void cancel(InterpreterContext context) {
+    if (iPythonInterpreter != null) {
+      iPythonInterpreter.cancel(context);
+    }
     try {
       interrupt();
     } catch (IOException e) {
@@ -425,11 +483,17 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
   @Override
   public int getProgress(InterpreterContext context) {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.getProgress(context);
+    }
     return 0;
   }
 
   @Override
   public Scheduler getScheduler() {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.getScheduler();
+    }
     return SchedulerFactory.singleton().createOrGetFIFOScheduler(
         PythonInterpreter.class.getName() + this.hashCode());
   }
@@ -437,6 +501,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   @Override
   public List<InterpreterCompletion> completion(String buf, int cursor,
       InterpreterContext interpreterContext) {
+    if (iPythonInterpreter != null) {
+      return iPythonInterpreter.completion(buf, cursor, interpreterContext);
+    }
     return null;
   }
 
@@ -485,7 +552,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
       bootstrapCode += line + "\n";
     }
 
-    interpret(bootstrapCode, context);
+    try {
+      interpret(bootstrapCode, context);
+    } catch (InterpreterException e) {
+      throw new IOException(e);
+    }
   }
 
   public GUI getGui() {

python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java

@@ -22,6 +22,7 @@ import java.util.Properties;
 
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
@@ -42,7 +43,7 @@ public class PythonInterpreterPandasSql extends Interpreter {
     super(property);
   }
 
-  PythonInterpreter getPythonInterpreter() {
+  PythonInterpreter getPythonInterpreter() throws InterpreterException {
     LazyOpenInterpreter lazy = null;
     PythonInterpreter python = null;
     Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
@@ -62,7 +63,7 @@ public class PythonInterpreterPandasSql extends Interpreter {
   }
 
   @Override
-  public void open() {
+  public void open() throws InterpreterException {
     LOG.info("Open Python SQL interpreter instance: {}", this.toString());
 
     try {
@@ -76,14 +77,15 @@ public class PythonInterpreterPandasSql extends Interpreter {
   }
 
   @Override
-  public void close() {
+  public void close() throws InterpreterException {
     LOG.info("Close Python SQL interpreter instance: {}", this.toString());
     Interpreter python = getPythonInterpreter();
     python.close();
   }
 
   @Override
-  public InterpreterResult interpret(String st, InterpreterContext context) {
+  public InterpreterResult interpret(String st, InterpreterContext context)
+      throws InterpreterException {
     LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
     Interpreter python = getPythonInterpreter();
 

python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java

@@ -0,0 +1,49 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.zeppelin.python;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ZeppelinContext for Python
+ */
+public class PythonZeppelinContext extends BaseZeppelinContext {
+
+  public PythonZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+    super(hooks, maxResult);
+  }
+
+  @Override
+  public Map<String, String> getInterpreterClassMap() {
+    return null;
+  }
+
+  @Override
+  public List<Class> getSupportedClasses() {
+    return null;
+  }
+
+  @Override
+  protected String showData(Object obj) {
+    return null;
+  }
+}

python/src/main/proto/ipython.proto

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.apache.zeppelin.python.proto";
+option java_outer_classname = "IPythonProto";
+option objc_class_prefix = "IPython";
+
+package ipython;
+
+// The IPython service definition.
+service IPython {
+  // Sends code
+  rpc execute (ExecuteRequest) returns (stream ExecuteResponse) {}
+
+  // Get completion
+  rpc complete (CompletionRequest) returns (CompletionResponse) {}
+
+  // Cancel the running statement
+  rpc cancel (CancelRequest) returns (CancelResponse) {}
+
+  // Get ipython kernel status
+  rpc status (StatusRequest) returns (StatusResponse) {}
+
+  rpc stop(StopRequest) returns (StopResponse) {}
+}
+
+enum ExecuteStatus {
+  SUCCESS = 0;
+  ERROR = 1;
+}
+
+enum IPythonStatus {
+  STARTING = 0;
+  RUNNING = 1;
+}
+
+enum OutputType {
+  TEXT = 0;
+  IMAGE = 1;
+}
+
+// The request message containing the code
+message ExecuteRequest {
+  string code = 1;
+}
+
+// The response message containing the execution result.
+message ExecuteResponse {
+  ExecuteStatus status = 1;
+  OutputType type = 2;
+  string output = 3;
+}
+
+message CancelRequest {
+
+}
+
+message CancelResponse {
+
+}
+
+message CompletionRequest {
+  string code = 1;
+  int32 cursor = 2;
+}
+
+message CompletionResponse {
+  repeated string matches = 1;
+}
+
+message StatusRequest {
+
+}
+
+message StatusResponse {
+  IPythonStatus status = 1;
+}
+
+message StopRequest {
+
+}
+
+message StopResponse {
+
+}

python/src/main/resources/grpc/generate_rpc.sh

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+python -m grpc_tools.protoc -I../../proto --python_out=python --grpc_python_out=python ../../proto/ipython.proto

python/src/main/resources/grpc/python/ipython_client.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import grpc
+
+import ipython_pb2
+import ipython_pb2_grpc
+
+
+def run():
+    channel = grpc.insecure_channel('localhost:50053')
+    stub = ipython_pb2_grpc.IPythonStub(channel)
+    response = stub.execute(ipython_pb2.ExecuteRequest(code="import time\nfor i in range(1,4):\n\ttime.sleep(1)\n\tprint(i)\n" +
+                                                            "%matplotlib inline\nimport matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)"))
+    for r in response:
+        print("output:" + r.output)
+
+    response = stub.execute(ipython_pb2.ExecuteRequest(code="range?"))
+    for r in response:
+        print(r)
+
+if __name__ == '__main__':
+    run()

python/src/main/resources/grpc/python/ipython_pb2.py

@@ -0,0 +1,751 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: ipython.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='ipython.proto',
+  package='ipython',
+  syntax='proto3',
+  serialized_pb=_b('\n\ripython.proto\x12\x07ipython\"\x1e\n\x0e\x45xecuteRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\"l\n\x0f\x45xecuteResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.ExecuteStatus\x12!\n\x04type\x18\x02 \x01(\x0e\x32\x13.ipython.OutputType\x12\x0e\n\x06output\x18\x03 \x01(\t\"\x0f\n\rCancelRequest\"\x10\n\x0e\x43\x61ncelResponse\"1\n\x11\x43ompletionRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0e\n\x06\x63ursor\x18\x02 \x01(\x05\"%\n\x12\x43ompletionResponse\x12\x0f\n\x07matches\x18\x01 \x03(\t\"\x0f\n\rStatusRequest\"8\n\x0eStatusResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.IPythonStatus\"\r\n\x0bStopRequest\"\x0e\n\x0cStopResponse*\'\n\rExecuteStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\t\n\x05\x45RROR\x10\x01**\n\rIPythonStatus\x12\x0c\n\x08STARTING\x10\x00\x12\x0b\n\x07RUNNING\x10\x01*!\n\nOutputType\x12\x08\n\x04TEXT\x10\x00\x12\t\n\x05IMAGE\x10\x01\x32\xc3\x02\n\x07IPython\x12@\n\x07\x65xecute\x12\x17.ipython.ExecuteRequest\x1a\x18.ipython.ExecuteResponse\"\x00\x30\x01\x12\x45\n\x08\x63omplete\x12\x1a.ipython.CompletionRequest\x1a\x1b.ipython.CompletionResponse\"\x00\x12;\n\x06\x63\x61ncel\x12\x16.ipython.CancelRequest\x1a\x17.ipython.CancelResponse\"\x00\x12;\n\x06status\x12\x16.ipython.StatusRequest\x1a\x17.ipython.StatusResponse\"\x00\x12\x35\n\x04stop\x12\x14.ipython.StopRequest\x1a\x15.ipython.StopResponse\"\x00\x42<\n org.apache.zeppelin.python.protoB\x0cIPythonProtoP\x01\xa2\x02\x07IPythonb\x06proto3')
+)
+
+_EXECUTESTATUS = _descriptor.EnumDescriptor(
+  name='ExecuteStatus',
+  full_name='ipython.ExecuteStatus',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='SUCCESS', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='ERROR', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=399,
+  serialized_end=438,
+)
+_sym_db.RegisterEnumDescriptor(_EXECUTESTATUS)
+
+ExecuteStatus = enum_type_wrapper.EnumTypeWrapper(_EXECUTESTATUS)
+_IPYTHONSTATUS = _descriptor.EnumDescriptor(
+  name='IPythonStatus',
+  full_name='ipython.IPythonStatus',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='STARTING', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='RUNNING', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=440,
+  serialized_end=482,
+)
+_sym_db.RegisterEnumDescriptor(_IPYTHONSTATUS)
+
+IPythonStatus = enum_type_wrapper.EnumTypeWrapper(_IPYTHONSTATUS)
+_OUTPUTTYPE = _descriptor.EnumDescriptor(
+  name='OutputType',
+  full_name='ipython.OutputType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='TEXT', index=0, number=0,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='IMAGE', index=1, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=484,
+  serialized_end=517,
+)
+_sym_db.RegisterEnumDescriptor(_OUTPUTTYPE)
+
+OutputType = enum_type_wrapper.EnumTypeWrapper(_OUTPUTTYPE)
+SUCCESS = 0
+ERROR = 1
+STARTING = 0
+RUNNING = 1
+TEXT = 0
+IMAGE = 1
+
+
+
+_EXECUTEREQUEST = _descriptor.Descriptor(
+  name='ExecuteRequest',
+  full_name='ipython.ExecuteRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='ipython.ExecuteRequest.code', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=26,
+  serialized_end=56,
+)
+
+
+_EXECUTERESPONSE = _descriptor.Descriptor(
+  name='ExecuteResponse',
+  full_name='ipython.ExecuteResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='status', full_name='ipython.ExecuteResponse.status', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='type', full_name='ipython.ExecuteResponse.type', index=1,
+      number=2, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='output', full_name='ipython.ExecuteResponse.output', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=58,
+  serialized_end=166,
+)
+
+
+_CANCELREQUEST = _descriptor.Descriptor(
+  name='CancelRequest',
+  full_name='ipython.CancelRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=168,
+  serialized_end=183,
+)
+
+
+_CANCELRESPONSE = _descriptor.Descriptor(
+  name='CancelResponse',
+  full_name='ipython.CancelResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=185,
+  serialized_end=201,
+)
+
+
+_COMPLETIONREQUEST = _descriptor.Descriptor(
+  name='CompletionRequest',
+  full_name='ipython.CompletionRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='ipython.CompletionRequest.code', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='cursor', full_name='ipython.CompletionRequest.cursor', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=203,
+  serialized_end=252,
+)
+
+
+_COMPLETIONRESPONSE = _descriptor.Descriptor(
+  name='CompletionResponse',
+  full_name='ipython.CompletionResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='matches', full_name='ipython.CompletionResponse.matches', index=0,
+      number=1, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=254,
+  serialized_end=291,
+)
+
+
+_STATUSREQUEST = _descriptor.Descriptor(
+  name='StatusRequest',
+  full_name='ipython.StatusRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=293,
+  serialized_end=308,
+)
+
+
+_STATUSRESPONSE = _descriptor.Descriptor(
+  name='StatusResponse',
+  full_name='ipython.StatusResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='status', full_name='ipython.StatusResponse.status', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=310,
+  serialized_end=366,
+)
+
+
+_STOPREQUEST = _descriptor.Descriptor(
+  name='StopRequest',
+  full_name='ipython.StopRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=368,
+  serialized_end=381,
+)
+
+
+_STOPRESPONSE = _descriptor.Descriptor(
+  name='StopResponse',
+  full_name='ipython.StopResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=383,
+  serialized_end=397,
+)
+
+_EXECUTERESPONSE.fields_by_name['status'].enum_type = _EXECUTESTATUS
+_EXECUTERESPONSE.fields_by_name['type'].enum_type = _OUTPUTTYPE
+_STATUSRESPONSE.fields_by_name['status'].enum_type = _IPYTHONSTATUS
+DESCRIPTOR.message_types_by_name['ExecuteRequest'] = _EXECUTEREQUEST
+DESCRIPTOR.message_types_by_name['ExecuteResponse'] = _EXECUTERESPONSE
+DESCRIPTOR.message_types_by_name['CancelRequest'] = _CANCELREQUEST
+DESCRIPTOR.message_types_by_name['CancelResponse'] = _CANCELRESPONSE
+DESCRIPTOR.message_types_by_name['CompletionRequest'] = _COMPLETIONREQUEST
+DESCRIPTOR.message_types_by_name['CompletionResponse'] = _COMPLETIONRESPONSE
+DESCRIPTOR.message_types_by_name['StatusRequest'] = _STATUSREQUEST
+DESCRIPTOR.message_types_by_name['StatusResponse'] = _STATUSRESPONSE
+DESCRIPTOR.message_types_by_name['StopRequest'] = _STOPREQUEST
+DESCRIPTOR.message_types_by_name['StopResponse'] = _STOPRESPONSE
+DESCRIPTOR.enum_types_by_name['ExecuteStatus'] = _EXECUTESTATUS
+DESCRIPTOR.enum_types_by_name['IPythonStatus'] = _IPYTHONSTATUS
+DESCRIPTOR.enum_types_by_name['OutputType'] = _OUTPUTTYPE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ExecuteRequest = _reflection.GeneratedProtocolMessageType('ExecuteRequest', (_message.Message,), dict(
+  DESCRIPTOR = _EXECUTEREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.ExecuteRequest)
+  ))
+_sym_db.RegisterMessage(ExecuteRequest)
+
+ExecuteResponse = _reflection.GeneratedProtocolMessageType('ExecuteResponse', (_message.Message,), dict(
+  DESCRIPTOR = _EXECUTERESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.ExecuteResponse)
+  ))
+_sym_db.RegisterMessage(ExecuteResponse)
+
+CancelRequest = _reflection.GeneratedProtocolMessageType('CancelRequest', (_message.Message,), dict(
+  DESCRIPTOR = _CANCELREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CancelRequest)
+  ))
+_sym_db.RegisterMessage(CancelRequest)
+
+CancelResponse = _reflection.GeneratedProtocolMessageType('CancelResponse', (_message.Message,), dict(
+  DESCRIPTOR = _CANCELRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CancelResponse)
+  ))
+_sym_db.RegisterMessage(CancelResponse)
+
+CompletionRequest = _reflection.GeneratedProtocolMessageType('CompletionRequest', (_message.Message,), dict(
+  DESCRIPTOR = _COMPLETIONREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CompletionRequest)
+  ))
+_sym_db.RegisterMessage(CompletionRequest)
+
+CompletionResponse = _reflection.GeneratedProtocolMessageType('CompletionResponse', (_message.Message,), dict(
+  DESCRIPTOR = _COMPLETIONRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.CompletionResponse)
+  ))
+_sym_db.RegisterMessage(CompletionResponse)
+
+StatusRequest = _reflection.GeneratedProtocolMessageType('StatusRequest', (_message.Message,), dict(
+  DESCRIPTOR = _STATUSREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StatusRequest)
+  ))
+_sym_db.RegisterMessage(StatusRequest)
+
+StatusResponse = _reflection.GeneratedProtocolMessageType('StatusResponse', (_message.Message,), dict(
+  DESCRIPTOR = _STATUSRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StatusResponse)
+  ))
+_sym_db.RegisterMessage(StatusResponse)
+
+StopRequest = _reflection.GeneratedProtocolMessageType('StopRequest', (_message.Message,), dict(
+  DESCRIPTOR = _STOPREQUEST,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StopRequest)
+  ))
+_sym_db.RegisterMessage(StopRequest)
+
+StopResponse = _reflection.GeneratedProtocolMessageType('StopResponse', (_message.Message,), dict(
+  DESCRIPTOR = _STOPRESPONSE,
+  __module__ = 'ipython_pb2'
+  # @@protoc_insertion_point(class_scope:ipython.StopResponse)
+  ))
+_sym_db.RegisterMessage(StopResponse)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n org.apache.zeppelin.python.protoB\014IPythonProtoP\001\242\002\007IPython'))
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+
+
+  class IPythonStub(object):
+    """The IPython service definition.
+    """
+
+    def __init__(self, channel):
+      """Constructor.
+
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.execute = channel.unary_stream(
+          '/ipython.IPython/execute',
+          request_serializer=ExecuteRequest.SerializeToString,
+          response_deserializer=ExecuteResponse.FromString,
+          )
+      self.complete = channel.unary_unary(
+          '/ipython.IPython/complete',
+          request_serializer=CompletionRequest.SerializeToString,
+          response_deserializer=CompletionResponse.FromString,
+          )
+      self.cancel = channel.unary_unary(
+          '/ipython.IPython/cancel',
+          request_serializer=CancelRequest.SerializeToString,
+          response_deserializer=CancelResponse.FromString,
+          )
+      self.status = channel.unary_unary(
+          '/ipython.IPython/status',
+          request_serializer=StatusRequest.SerializeToString,
+          response_deserializer=StatusResponse.FromString,
+          )
+      self.stop = channel.unary_unary(
+          '/ipython.IPython/stop',
+          request_serializer=StopRequest.SerializeToString,
+          response_deserializer=StopResponse.FromString,
+          )
+
+
+  class IPythonServicer(object):
+    """The IPython service definition.
+    """
+
+    def execute(self, request, context):
+      """Sends code
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def complete(self, request, context):
+      """Get completion
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def cancel(self, request, context):
+      """Cancel the running statement
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def status(self, request, context):
+      """Get ipython kernel status
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def stop(self, request, context):
+      # missing associated documentation comment in .proto file
+      pass
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_IPythonServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'execute': grpc.unary_stream_rpc_method_handler(
+            servicer.execute,
+            request_deserializer=ExecuteRequest.FromString,
+            response_serializer=ExecuteResponse.SerializeToString,
+        ),
+        'complete': grpc.unary_unary_rpc_method_handler(
+            servicer.complete,
+            request_deserializer=CompletionRequest.FromString,
+            response_serializer=CompletionResponse.SerializeToString,
+        ),
+        'cancel': grpc.unary_unary_rpc_method_handler(
+            servicer.cancel,
+            request_deserializer=CancelRequest.FromString,
+            response_serializer=CancelResponse.SerializeToString,
+        ),
+        'status': grpc.unary_unary_rpc_method_handler(
+            servicer.status,
+            request_deserializer=StatusRequest.FromString,
+            response_serializer=StatusResponse.SerializeToString,
+        ),
+        'stop': grpc.unary_unary_rpc_method_handler(
+            servicer.stop,
+            request_deserializer=StopRequest.FromString,
+            response_serializer=StopResponse.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'ipython.IPython', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaIPythonServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The IPython service definition.
+    """
+    def execute(self, request, context):
+      """Sends code
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def complete(self, request, context):
+      """Get completion
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def cancel(self, request, context):
+      """Cancel the running statement
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def status(self, request, context):
+      """Get ipython kernel status
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def stop(self, request, context):
+      # missing associated documentation comment in .proto file
+      pass
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+  class BetaIPythonStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The IPython service definition.
+    """
+    def execute(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Sends code
+      """
+      raise NotImplementedError()
+    def complete(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Get completion
+      """
+      raise NotImplementedError()
+    complete.future = None
+    def cancel(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Cancel the running statement
+      """
+      raise NotImplementedError()
+    cancel.future = None
+    def status(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Get ipython kernel status
+      """
+      raise NotImplementedError()
+    status.future = None
+    def stop(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      # missing associated documentation comment in .proto file
+      pass
+      raise NotImplementedError()
+    stop.future = None
+
+
+  def beta_create_IPython_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('ipython.IPython', 'cancel'): CancelRequest.FromString,
+      ('ipython.IPython', 'complete'): CompletionRequest.FromString,
+      ('ipython.IPython', 'execute'): ExecuteRequest.FromString,
+      ('ipython.IPython', 'status'): StatusRequest.FromString,
+      ('ipython.IPython', 'stop'): StopRequest.FromString,
+    }
+    response_serializers = {
+      ('ipython.IPython', 'cancel'): CancelResponse.SerializeToString,
+      ('ipython.IPython', 'complete'): CompletionResponse.SerializeToString,
+      ('ipython.IPython', 'execute'): ExecuteResponse.SerializeToString,
+      ('ipython.IPython', 'status'): StatusResponse.SerializeToString,
+      ('ipython.IPython', 'stop'): StopResponse.SerializeToString,
+    }
+    method_implementations = {
+      ('ipython.IPython', 'cancel'): face_utilities.unary_unary_inline(servicer.cancel),
+      ('ipython.IPython', 'complete'): face_utilities.unary_unary_inline(servicer.complete),
+      ('ipython.IPython', 'execute'): face_utilities.unary_stream_inline(servicer.execute),
+      ('ipython.IPython', 'status'): face_utilities.unary_unary_inline(servicer.status),
+      ('ipython.IPython', 'stop'): face_utilities.unary_unary_inline(servicer.stop),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_IPython_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('ipython.IPython', 'cancel'): CancelRequest.SerializeToString,
+      ('ipython.IPython', 'complete'): CompletionRequest.SerializeToString,
+      ('ipython.IPython', 'execute'): ExecuteRequest.SerializeToString,
+      ('ipython.IPython', 'status'): StatusRequest.SerializeToString,
+      ('ipython.IPython', 'stop'): StopRequest.SerializeToString,
+    }
+    response_deserializers = {
+      ('ipython.IPython', 'cancel'): CancelResponse.FromString,
+      ('ipython.IPython', 'complete'): CompletionResponse.FromString,
+      ('ipython.IPython', 'execute'): ExecuteResponse.FromString,
+      ('ipython.IPython', 'status'): StatusResponse.FromString,
+      ('ipython.IPython', 'stop'): StopResponse.FromString,
+    }
+    cardinalities = {
+      'cancel': cardinality.Cardinality.UNARY_UNARY,
+      'complete': cardinality.Cardinality.UNARY_UNARY,
+      'execute': cardinality.Cardinality.UNARY_STREAM,
+      'status': cardinality.Cardinality.UNARY_UNARY,
+      'stop': cardinality.Cardinality.UNARY_UNARY,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'ipython.IPython', cardinalities, options=stub_options)
+except ImportError:
+  pass
+# @@protoc_insertion_point(module_scope)

python/src/main/resources/grpc/python/ipython_pb2_grpc.py

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+import ipython_pb2 as ipython__pb2
+
+
+class IPythonStub(object):
+  """The IPython service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.execute = channel.unary_stream(
+        '/ipython.IPython/execute',
+        request_serializer=ipython__pb2.ExecuteRequest.SerializeToString,
+        response_deserializer=ipython__pb2.ExecuteResponse.FromString,
+        )
+    self.complete = channel.unary_unary(
+        '/ipython.IPython/complete',
+        request_serializer=ipython__pb2.CompletionRequest.SerializeToString,
+        response_deserializer=ipython__pb2.CompletionResponse.FromString,
+        )
+    self.cancel = channel.unary_unary(
+        '/ipython.IPython/cancel',
+        request_serializer=ipython__pb2.CancelRequest.SerializeToString,
+        response_deserializer=ipython__pb2.CancelResponse.FromString,
+        )
+    self.status = channel.unary_unary(
+        '/ipython.IPython/status',
+        request_serializer=ipython__pb2.StatusRequest.SerializeToString,
+        response_deserializer=ipython__pb2.StatusResponse.FromString,
+        )
+    self.stop = channel.unary_unary(
+        '/ipython.IPython/stop',
+        request_serializer=ipython__pb2.StopRequest.SerializeToString,
+        response_deserializer=ipython__pb2.StopResponse.FromString,
+        )
+
+
+class IPythonServicer(object):
+  """The IPython service definition.
+  """
+
+  def execute(self, request, context):
+    """Sends code
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def complete(self, request, context):
+    """Get completion
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def cancel(self, request, context):
+    """Cancel the running statement
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def status(self, request, context):
+    """Get ipython kernel status
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def stop(self, request, context):
+    # missing associated documentation comment in .proto file
+    pass
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_IPythonServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'execute': grpc.unary_stream_rpc_method_handler(
+          servicer.execute,
+          request_deserializer=ipython__pb2.ExecuteRequest.FromString,
+          response_serializer=ipython__pb2.ExecuteResponse.SerializeToString,
+      ),
+      'complete': grpc.unary_unary_rpc_method_handler(
+          servicer.complete,
+          request_deserializer=ipython__pb2.CompletionRequest.FromString,
+          response_serializer=ipython__pb2.CompletionResponse.SerializeToString,
+      ),
+      'cancel': grpc.unary_unary_rpc_method_handler(
+          servicer.cancel,
+          request_deserializer=ipython__pb2.CancelRequest.FromString,
+          response_serializer=ipython__pb2.CancelResponse.SerializeToString,
+      ),
+      'status': grpc.unary_unary_rpc_method_handler(
+          servicer.status,
+          request_deserializer=ipython__pb2.StatusRequest.FromString,
+          response_serializer=ipython__pb2.StatusResponse.SerializeToString,
+      ),
+      'stop': grpc.unary_unary_rpc_method_handler(
+          servicer.stop,
+          request_deserializer=ipython__pb2.StopRequest.FromString,
+          response_serializer=ipython__pb2.StopResponse.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'ipython.IPython', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))