Merge branch 'master' of https://github.com/apache/zeppelin

.travis.yml

@@ -34,49 +34,61 @@ addons:
     packages:
     - r-base-dev
 
+env:
+  global:
+    # Interpreters does not required by zeppelin-server integration tests
+    - INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy'
+
 matrix:
   include:
     # Test License compliance using RAT tool
     - jdk: "oraclejdk7"
       env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
 
-    # Test all modules with spark 2.1.0 and scala 2.11
-    - sudo: required
-      jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
-
-    # Test all modules with spark 2.0.2 and scala 2.11
+    # Test core modules
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
-
-    # Test spark module for 1.6.3 with scala 2.10
-    - sudo: required
-      jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10 -Plivy-0.2" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
-
-    # Test spark module for 1.6.3 with scala 2.11
-    - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11 -Dscala.version=2.11.7 -Dscala.binary.version=2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false"
 
     # Test selenium with spark module for 1.6.3
     - jdk: "oraclejdk7"
-      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+      env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
+
+    # Test interpreter modules
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+
+    # Test spark module for 2.1.0 with scala 2.11, livy
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
+
+    # Test spark module for 2.0.2 with scala 2.11
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+
+    # Test spark module for 1.6.3 with scala 2.10
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+
+    # Test spark module for 1.6.3 with scala 2.11
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test python/pyspark with python 2
     - jdk: "oraclejdk7"
-      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
     # Test python/pyspark with python 3
     - jdk: "oraclejdk7"
-      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
+      env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
 
 before_install:
   # check files included in commit range, clear bower_components if a bower.json file has changed.
   # bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message
-  - changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE)
+  - changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE 2>/dev/null) || changedfiles=""
   - echo $changedfiles
   - hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true);
-  - clearcache=$(git log $TRAVIS_COMMIT_RANGE | grep -c -E "clear bower|bower clear" || true)
+  - gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog="" 
+  - clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
   - if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache clear; else echo "Using cached bower_components."; fi
   - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
   - ./testing/install_external_dependencies.sh
@@ -90,7 +102,7 @@ install:
   - mvn $BUILD_FLAG $MODULES $PROFILE -B
 
 before_script:
-  - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
+  - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
   - if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
   - if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
   - if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi

bin/interpreter.sh

@@ -20,10 +20,10 @@ bin=$(dirname "${BASH_SOURCE-$0}")
 bin=$(cd "${bin}">/dev/null; pwd)
 
 function usage() {
-    echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
+    echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
 }
 
-while getopts "hp:d:l:v:u:" o; do
+while getopts "hp:d:l:v:u:g:" o; do
     case ${o} in
         h)
             usage
@@ -50,6 +50,9 @@ while getopts "hp:d:l:v:u:" o; do
               ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
             fi
             ;;
+        g)
+            INTERPRETER_GROUP_NAME=${OPTARG}
+            ;;
         esac
 done
 
@@ -86,6 +89,9 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
 INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
 ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
 ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
+if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then
+    ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-"
+fi
 if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
     ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
 fi

conf/zeppelin-env.cmd.template

@@ -38,6 +38,8 @@ REM set ZEPPELIN_NOTEBOOK_S3_SSE               REM Server-side encryption enable
 REM set ZEPPELIN_IDENT_STRING   		REM A string representing this instance of zeppelin. $USER by default.
 REM set ZEPPELIN_NICENESS       		REM The scheduling priority for daemons. Defaults to 0.
 REM set ZEPPELIN_INTERPRETER_LOCALREPO         REM Local repository for interpreter's additional dependency loading
+REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO       REM Maven principal repository for interpreter's additional dependency loading
+REM set ZEPPELIN_HELIUM_NPM_REGISTRY           REM Remote Npm registry for Helium dependency loader
 REM set ZEPPELIN_NOTEBOOK_STORAGE		REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
 REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC		REM If there are multiple notebook storages, should we treat the first one as the only source of truth?
 

conf/zeppelin-env.sh.template

@@ -43,6 +43,8 @@
 # export ZEPPELIN_IDENT_STRING   		# A string representing this instance of zeppelin. $USER by default.
 # export ZEPPELIN_NICENESS       		# The scheduling priority for daemons. Defaults to 0.
 # export ZEPPELIN_INTERPRETER_LOCALREPO         # Local repository for interpreter's additional dependency loading
+# export ZEPPELIN_INTERPRETER_DEP_MVNREPO       # Remote principal repository for interpreter's additional dependency loading
+# export ZEPPELIN_HELIUM_NPM_REGISTRY           # Remote Npm registry for Helium dependency loader
 # export ZEPPELIN_NOTEBOOK_STORAGE 		# Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
 # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC	# If there are multiple notebook storages, should we treat the first one as the only source of truth?
 # export ZEPPELIN_NOTEBOOK_PUBLIC   # Make notebook public by default when created, private otherwise

conf/zeppelin-site.xml.template

@@ -239,12 +239,24 @@
   <description>Local repository for interpreter's additional dependency loading</description>
 </property>
 
+<property>
+  <name>zeppelin.interpreter.dep.mvnRepo</name>
+  <value>http://repo1.maven.org/maven2/</value>
+  <description>Remote principal repository for interpreter's additional dependency loading</description>
+</property>
+
 <property>
   <name>zeppelin.dep.localrepo</name>
   <value>local-repo</value>
   <description>Local repository for dependency loader</description>
 </property>
 
+<property>
+  <name>zeppelin.helium.npm.registry</name>
+  <value>http://registry.npmjs.org/</value>
+  <description>Remote Npm registry for Helium dependency loader</description>
+</property>
+
 <property>
   <name>zeppelin.interpreters</name>
   <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>

dev/common_release.sh

@@ -33,6 +33,9 @@ fi
 
 mkdir "${WORKING_DIR}"
 
+# If set to 'yes', release script will deploy artifacts to SNAPSHOT repository.
+DO_SNAPSHOT='no'
+
 usage() {
   echo "usage) $0 [Release version] [Branch or Tag]"
   echo "   ex. $0 0.6.0 v0.6.0"

dev/publish_release.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -43,6 +42,9 @@ NC='\033[0m' # No Color
 
 RELEASE_VERSION="$1"
 GIT_TAG="$2"
+if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
+  DO_SNAPSHOT="yes"
+fi
 
 PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
 PROJECT_OPTIONS="-pl !zeppelin-distribution"
@@ -67,6 +69,40 @@ function curl_error() {
   fi
 }
 
+
+#
+# Publishing Apache Zeppelin artifact to Apache snapshot repository.
+#
+function publish_snapshot_to_maven() {
+  cd "${WORKING_DIR}/zeppelin"
+  echo "Deploying Apache Zeppelin $RELEASE_VERSION version to snapshot repository."
+
+  if [[ ! $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
+    echo "ERROR: Snapshots must have a version containing 'SNAPSHOT'"
+    echo "ERROR: You gave version '$RELEASE_VERSION'"
+    exit 1
+  fi
+
+  tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)"
+
+  mvn versions:set -DnewVersion=$RELEASE_VERSION
+  tmp_settings="tmp-settings.xml"
+  echo "<settings><servers><server>" > $tmp_settings
+  echo "<id>apache.snapshots.https</id><username>$ASF_USERID</username>" >> $tmp_settings
+  echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
+  echo "</server></servers></settings>" >> $tmp_settings
+
+  mvn --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \
+    $PUBLISH_PROFILES -Drat.skip=true deploy
+
+  "${BASEDIR}/change_scala_version.sh" 2.11
+  mvn -Pscala-2.11 --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \
+    $PUBLISH_PROFILES -Drat.skip=true clean deploy
+
+  rm $tmp_settings
+  rm -rf $tmp_repo
+}
+
 function publish_to_maven() {
   cd "${WORKING_DIR}/zeppelin"
 
@@ -153,5 +189,9 @@ function publish_to_maven() {
 }
 
 git_clone
-publish_to_maven
+if [[ "${DO_SNAPSHOT}" == 'yes' ]]; then
+  publish_snapshot_to_maven
+else
+  publish_to_maven
+fi
 cleanup

dev/tag_release.sh

@@ -44,7 +44,7 @@ git tag "${RC_TAG}"
 git commit -a -m "Preparing development version ${NEXT_DEV_VERSION}"
 
 git push origin "${RC_TAG}"
-git push origin HEAD:"${BRANCH}" 
+git push origin HEAD:"${GIT_BRANCH}" 
 
 popd
 rm -rf "${WORKING_DIR}"

docs/install/configuration.md

@@ -26,7 +26,7 @@ limitations under the License.
 ## Zeppelin Properties
 There are two locations you can configure Apache Zeppelin.
 
-* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows). 
+* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
 * **Java properties** can ba defined in `conf/zeppelin-site.xml`.
 
 If both are defined, then the **environment variables** will take priority.
@@ -44,7 +44,7 @@ If both are defined, then the **environment variables** will take priority.
     <td><h6 class="properties">zeppelin.server.port</h6></td>
     <td>8080</td>
     <td>Zeppelin server port </br>
-      <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with 
+      <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with
       <a href="https://zeppelin.apache.org/contribution/webapplication.html#dev-mode" target="_blank">Zeppelin web application development port</a> (default: 9000).</span></td>
   </tr>
   <tr>
@@ -257,6 +257,12 @@ If both are defined, then the **environment variables** will take priority.
     <td>interpreter</td>
     <td>Interpreter directory</td>
   </tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_INTERPRETER_DEP_MVNREPO</h6></td>
+    <td><h6 class="properties">zeppelin.interpreter.dep.mvnRepo</h6></td>
+    <td>http://repo1.maven.org/maven2/</td>
+    <td>Remote principal repository for interpreter's additional dependency loading</td>
+  </tr>
   <tr>
     <td><h6 class="properties">ZEPPELIN_INTERPRETER_OUTPUT_LIMIT</h6></td>
     <td><h6 class="properties">zeppelin.interpreter.output.limit</h6></td>
@@ -275,6 +281,12 @@ If both are defined, then the **environment variables** will take priority.
     <td>local-repo</td>
     <td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td>
   </tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td>
+    <td><h6 class="properties">zeppelin.helium.npm.registry</h6></td>
+    <td>http://registry.npmjs.org/</td>
+    <td>Remote Npm registry for Helium dependency loader</td>
+  </tr>
   <tr>
     <td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td>
     <td><h6 class="properties">zeppelin.websocket.max.text.message.size</h6></td>
@@ -391,7 +403,7 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order
 ### Obfuscating Passwords using the Jetty Password Tool
 
 Security best practices advise to not use plain text passwords and Jetty provides a password tool to help obfuscating the passwords used to access the KeyStore and TrustStore.
- 
+
 The Password tool documentation can be found [here](http://www.eclipse.org/jetty/documentation/current/configuring-security-secure-passwords.html).
 
 After using the tool:

docs/interpreter/jdbc.md

@@ -118,6 +118,11 @@ The JDBC interpreter properties are defined by default like below.
     <td>gpadmin</td>
     <td>The JDBC user name</td>
   </tr>
+  <tr>
+    <td>default.precode</td>
+    <td></td>
+    <td>Some SQL which executes while opening connection</td>
+  </tr>
 </table>
 
 If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
@@ -167,10 +172,6 @@ There are more JDBC interpreter properties you can specify like below.
     <td>default.jceks.credentialKey</td>
     <td>jceks credential key</td>
   </tr>
-  <tr>
-    <td>zeppelin.jdbc.precode</td>
-    <td>Some SQL which executes while opening connection</td>
-  </tr>
 </table>
 
 You can also add more properties by using this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29).
@@ -221,6 +222,75 @@ SELECT name, country, performer
 FROM demo.performers
 WHERE name='{{"{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia"}}}}'
 ```
+### Usage *precode*
+You can set *precode* for each data source. Code runs once while opening the connection.
+
+##### Properties
+An example settings of interpreter for the two data sources, each of which has its *precode* parameter.
+
+<table class="table-configuration">
+  <tr>
+    <th>Property Name</th>
+    <th>Value</th>
+  </tr>
+  <tr>
+    <td>default.driver</td>
+    <td>org.postgresql.Driver</td>
+  </tr>
+  <tr>
+    <td>default.password</td>
+    <td>1</td>
+  </tr>
+  <tr>
+    <td>default.url</td>
+    <td>jdbc:postgresql://localhost:5432/</td>
+  </tr>
+  <tr>
+    <td>default.user</td>
+    <td>postgres</td>
+  </tr>
+  <tr>
+    <td>default.precode</td>
+    <td>set search_path='test_path'</td>
+  </tr>
+  <tr>
+    <td>mysql.driver</td>
+    <td>com.mysql.jdbc.Driver</td>
+  </tr>
+  <tr>
+    <td>mysql.password</td>
+    <td>1</td>
+  </tr>
+  <tr>
+    <td>mysql.url</td>
+    <td>jdbc:mysql://localhost:3306/</td>
+  </tr>
+  <tr>
+    <td>mysql.user</td>
+    <td>root</td>
+  </tr>
+  <tr>
+    <td>mysql.precode</td>
+    <td>set @v=12</td>
+  </tr>
+</table>
+
+##### Usage
+Test of execution *precode* for each data source. 
+
+```sql
+%jdbc
+show search_path
+```
+Returns value of `search_path` which is set in the *default.precode*.
+
+
+```sql
+%jdbc(mysql)
+select @v
+```
+Returns value of `v` which is set in the *mysql.precode*.
+
 
 ## Examples
 Here are some examples you can refer to. Including the below connectors, you can connect every databases as long as it can be configured with it's JDBC driver.

docs/interpreter/livy.md

@@ -130,6 +130,16 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
       <td></td>
       <td>Adding extra libraries to livy interpreter</td>
     </tr>
+  <tr>
+    <td>zeppelin.livy.ssl.trustStore</td>
+    <td></td>
+    <td>client trustStore file. Used when livy ssl is enabled</td>
+  </tr>
+  <tr>
+    <td>zeppelin.livy.ssl.trustStorePassword</td>
+    <td></td>
+    <td>password for trustStore file. Used when livy ssl is enabled</td>
+  </tr>  
 </table>
 
 **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.**

docs/manual/dynamicform.md

@@ -56,7 +56,13 @@ Also you can separate option's display name and value, using `${formName=default
 
 <img src="../assets/themes/zeppelin/img/screenshots/form_select_displayname.png" />
 
-Hit enter after selecting option to run the paragraph with new value.
+The paragraph will be automatically run after you change your selection by default.
+But in case you have multiple types dynamic form in one paragraph, you might want to run the paragraph after changing all the selections.
+You can control this by unchecking the below **Run on selection change** option in the setting menu.
+
+Even if you uncheck this option, still you can run it by pressing `Enter`.
+
+<img src="../assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png" />
 
 ### Checkbox form
 
@@ -68,6 +74,14 @@ Besides, you can specify the delimiter using `${checkbox(delimiter):formName=...
 
 <img src="../assets/themes/zeppelin/img/screenshots/form_checkbox_delimiter.png">
 
+Like [select form](#select-form), the paragraph will be automatically run after you change your selection by default.
+But in case you have multiple types dynamic form in one paragraph, you might want to run the paragraph after changing all the selections.
+You can control this by unchecking the below **Run on selection change** option in the setting menu.
+
+Even if you uncheck this option, still you can run it by pressing `Enter`.
+
+<img src="../assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png" />
+
 ## Creates Programmatically
 
 Some language backend uses programmatic way to create form. For example [ZeppelinContext](../interpreter/spark.html#zeppelincontext) provides form creation API

docs/manual/interpreters.md

@@ -41,8 +41,39 @@ Zeppelin interpreter setting is the configuration of a given interpreter on Zepp
 
 <img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting.png" width="500px">
 
-Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
+Properties are exported as environment variables when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property. 
 
+You may use parameters from the context of interpreter by add #{contextParameterName} in value, parameter can be of the following types: string, number, boolean.
+
+###### Context parameters
+<table class="table-configuration">
+  <tr>
+    <th>Name</th>
+    <th>Type</th>
+  </tr>
+  <tr>
+    <td>user</td>
+    <td>string</td>
+  </tr>
+  <tr>
+    <td>noteId</td>
+    <td>string</td>
+  </tr>
+  <tr>
+    <td>replName</td>
+    <td>string</td>
+  </tr>
+  <tr>
+    <td>className</td>
+    <td>string</td>
+  </tr>
+</table>
+
+If context parameter is null then replaced by empty string.
+
+<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting_with_context_parameters.png" width="800px">
+
+<br>
 Each notebook can be bound to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
 
 <img src="../assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">

docs/rest-api/rest-notebook.md

@@ -111,7 +111,19 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
     },
     {
       "title": "paragraph title2",
-      "text": "paragraph text2"
+      "text": "paragraph text2",
+      "config": {
+        "title": true,
+        "colWidth": 6.0,
+        "results": [
+          {
+            "graph": {
+              "mode": "scatterChart",
+              "optionOpen": true
+            }
+          }
+        ]
+      }
     }
   ]
 }</pre></td>
@@ -598,6 +610,26 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
   "title": "Paragraph insert revised",
   "text": "%spark\nprintln(\"Paragraph insert revised\")",
   "index": 0
+}</pre></td>
+    </tr>
+    <tr>
+      <td> sample JSON input (providing paragraph config) </td>
+      <td><pre>
+{
+  "title": "paragraph title2",
+  "text": "paragraph text2",
+  "config": {
+    "title": true,
+    "colWidth": 6.0,
+    "results": [
+      {
+        "graph": {
+          "mode": "pieChart",
+          "optionOpen": true
+        }
+      }
+    ]
+  }
 }</pre></td>
     </tr>
     <tr>

interpreter/lib/python/backend_zinline.py

@@ -18,6 +18,7 @@
 
 from __future__ import print_function
 
+import sys
 import uuid
 import warnings
 import base64
@@ -94,7 +95,10 @@ class FigureCanvasZInline(FigureCanvasAgg):
         buf = BytesIO()
         self.print_figure(buf, **kwargs)
         fmt = fmt.encode()
-        byte_str = b"data:image/%s;base64," %fmt
+        if sys.version_info >= (3, 4) and sys.version_info < (3, 5):
+            byte_str = bytes("data:image/%s;base64," %fmt, "utf-8")
+        else:
+            byte_str = b"data:image/%s;base64," %fmt
         byte_str += base64.b64encode(buf.getvalue())
             
         # Python3 forces all strings to default to unicode, but for raster image

jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java

@@ -101,9 +101,10 @@ public class JDBCInterpreter extends Interpreter {
   static final String URL_KEY = "url";
   static final String USER_KEY = "user";
   static final String PASSWORD_KEY = "password";
+  static final String PRECODE_KEY = "precode";
   static final String JDBC_JCEKS_FILE = "jceks.file";
   static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
-  static final String ZEPPELIN_JDBC_PRECODE_KEY = "zeppelin.jdbc.precode";
+  static final String PRECODE_KEY_TEMPLATE = "%s.precode";
   static final String DOT = ".";
 
   private static final char WHITESPACE = ' ';
@@ -118,6 +119,7 @@ public class JDBCInterpreter extends Interpreter {
   static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
   static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
   static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
+  static final String DEFAULT_PRECODE = DEFAULT_KEY + DOT + PRECODE_KEY;
 
   static final String EMPTY_COLUMN_VALUE = "";
 
@@ -342,7 +344,7 @@ public class JDBCInterpreter extends Interpreter {
     if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
       createConnectionPool(url, user, propertyKey, properties);
       try (Connection connection = DriverManager.getConnection(jdbcDriver)) {
-        executePrecode(connection);
+        executePrecode(connection, propertyKey);
       }
     }
     return DriverManager.getConnection(jdbcDriver);
@@ -548,8 +550,8 @@ public class JDBCInterpreter extends Interpreter {
     return queries;
   }
 
-  private void executePrecode(Connection connection) throws SQLException {
-    String precode = getProperty(ZEPPELIN_JDBC_PRECODE_KEY);
+  private void executePrecode(Connection connection, String propertyKey) throws SQLException {
+    String precode = getProperty(String.format(PRECODE_KEY_TEMPLATE, propertyKey));
     if (StringUtils.isNotBlank(precode)) {
       precode = StringUtils.trim(precode);
       logger.info("Run SQL precode '{}'", precode);
@@ -633,7 +635,7 @@ public class JDBCInterpreter extends Interpreter {
         } catch (SQLException e) { /*ignored*/ }
       }
       getJDBCConfiguration(user).removeStatement(paragraphId);
-    } catch (Exception e) {
+    } catch (Throwable e) {
       if (e.getCause() instanceof TTransportException &&
           Throwables.getStackTraceAsString(e).contains("GSS") &&
           getJDBCConfiguration(user).isConnectionInDBDriverPoolSuccessful(propertyKey)) {

jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java

@@ -111,6 +111,8 @@ public class SqlCompleter extends StringsCompleter {
       try {
         while (schemas.next()) {
           String schemaName = schemas.getString("TABLE_SCHEM");
+          if (schemaName == null)
+            schemaName = "";
           if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
                   schemaFilter.replace("_", ".").replace("%", ".*?"))) {
             res.add(schemaName);

jdbc/src/main/resources/interpreter-setting.json

@@ -28,6 +28,12 @@
         "defaultValue": "org.postgresql.Driver",
         "description": "JDBC Driver Name"
       },
+      "default.precode": {
+        "envName": null,
+        "propertyName": "zeppelin.jdbc.precode",
+        "defaultValue": "",
+        "description": "SQL which executes while opening connection"
+      },
       "common.max_count": {
         "envName": null,
         "propertyName": "common.max_count",
@@ -63,12 +69,6 @@
         "propertyName": "zeppelin.jdbc.principal",
         "defaultValue": "",
         "description": "Kerberos principal"
-      },
-      "zeppelin.jdbc.precode": {
-        "envName": null,
-        "propertyName": "zeppelin.jdbc.precode",
-        "defaultValue": "",
-        "description": "SQL which executes while opening connection"
       }
     },
     "editor": {

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -19,6 +19,8 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
+import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE;
+import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE;
 import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
 import static org.junit.Assert.*;
 
@@ -44,8 +46,6 @@ import org.junit.Test;
 
 import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
 
-import static org.apache.zeppelin.jdbc.JDBCInterpreter.ZEPPELIN_JDBC_PRECODE_KEY;
-
 /**
  * JDBC interpreter unit tests
  */
@@ -397,7 +397,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
     properties.setProperty("default.url", getJdbcConnection());
     properties.setProperty("default.user", "");
     properties.setProperty("default.password", "");
-    properties.setProperty(ZEPPELIN_JDBC_PRECODE_KEY, "SET @testVariable=1");
+    properties.setProperty(DEFAULT_PRECODE, "SET @testVariable=1");
     JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
     jdbcInterpreter.open();
 
@@ -417,7 +417,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
     properties.setProperty("default.url", getJdbcConnection());
     properties.setProperty("default.user", "");
     properties.setProperty("default.password", "");
-    properties.setProperty(ZEPPELIN_JDBC_PRECODE_KEY, "incorrect command");
+    properties.setProperty(DEFAULT_PRECODE, "incorrect command");
     JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
     jdbcInterpreter.open();
 
@@ -428,4 +428,24 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
     assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
     assertEquals(InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType());
   }
- }
+
+  @Test
+  public void testPrecodeWithAnotherPrefix() throws SQLException, IOException {
+    Properties properties = new Properties();
+    properties.setProperty("anotherPrefix.driver", "org.h2.Driver");
+    properties.setProperty("anotherPrefix.url", getJdbcConnection());
+    properties.setProperty("anotherPrefix.user", "");
+    properties.setProperty("anotherPrefix.password", "");
+    properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "SET @testVariable=2");
+    JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
+    jdbcInterpreter.open();
+
+    String sqlQuery = "(anotherPrefix) select @testVariable";
+
+    InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
+
+    assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
+    assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
+    assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
+  }
+}

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java

@@ -21,6 +21,10 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContexts;
+import org.apache.http.impl.client.HttpClients;
 import org.apache.zeppelin.interpreter.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,11 +32,16 @@ import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
 import org.springframework.security.kerberos.client.KerberosRestTemplate;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
 
+import javax.net.ssl.SSLContext;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.security.KeyStore;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -56,10 +65,13 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   protected boolean displayAppInfo;
   private AtomicBoolean sessionExpired = new AtomicBoolean(false);
   protected LivyVersion livyVersion;
+  private RestTemplate restTemplate;
 
   // keep tracking the mapping between paragraphId and statementId, so that we can cancel the
   // statement after we execute it.
-  private ConcurrentHashMap<String, Integer> paragraphId2StmtIdMapping = new ConcurrentHashMap<>();
+  private ConcurrentHashMap<String, Integer> paragraphId2StmtIdMap = new ConcurrentHashMap<>();
+  private ConcurrentHashMap<String, Integer> paragraphId2StmtProgressMap =
+      new ConcurrentHashMap<>();
 
   public BaseLivyInterprereter(Properties property) {
     super(property);
@@ -70,6 +82,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
     this.pullStatusInterval = Integer.parseInt(
         property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
+    this.restTemplate = createRestTemplate();
   }
 
   public abstract String getSessionKind();
@@ -146,12 +159,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
           InterpreterUtils.getMostRelevantMessage(e));
     }
   }
-  
+
   @Override
   public void cancel(InterpreterContext context) {
     if (livyVersion.isCancelSupported()) {
       String paraId = context.getParagraphId();
-      Integer stmtId = paragraphId2StmtIdMapping.get(paraId);
+      Integer stmtId = paragraphId2StmtIdMap.get(paraId);
       try {
         if (stmtId != null) {
           cancelStatement(stmtId);
@@ -159,7 +172,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
       } catch (LivyException e) {
         LOGGER.error("Fail to cancel statement " + stmtId + " for paragraph " + paraId, e);
       } finally {
-        paragraphId2StmtIdMapping.remove(paraId);
+        paragraphId2StmtIdMap.remove(paraId);
       }
     } else {
       LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
@@ -173,6 +186,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
 
   @Override
   public int getProgress(InterpreterContext context) {
+    if (livyVersion.isGetProgressSupported()) {
+      String paraId = context.getParagraphId();
+      Integer progress = paragraphId2StmtProgressMap.get(paraId);
+      return progress == null ? 0 : progress;
+    }
     return 0;
   }
 
@@ -243,7 +261,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         stmtInfo = executeStatement(new ExecuteRequest(code));
       }
       if (paragraphId != null) {
-        paragraphId2StmtIdMapping.put(paragraphId, stmtInfo.id);
+        paragraphId2StmtIdMap.put(paragraphId, stmtInfo.id);
       }
       // pull the statement status
       while (!stmtInfo.isAvailable()) {
@@ -254,6 +272,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
           throw new LivyException(e);
         }
         stmtInfo = getStatementInfo(stmtInfo.id);
+        paragraphId2StmtProgressMap.put(paragraphId, (int) (stmtInfo.progress * 100));
       }
       if (appendSessionExpired) {
         return appendSessionExpire(getResultFromStatementInfo(stmtInfo, displayAppInfo),
@@ -263,7 +282,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
       }
     } finally {
       if (paragraphId != null) {
-        paragraphId2StmtIdMapping.remove(paragraphId);
+        paragraphId2StmtIdMap.remove(paragraphId);
+        paragraphId2StmtProgressMap.remove(paragraphId);
       }
     }
   }
@@ -312,12 +332,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
     } else {
       //TODO(zjffdu) support other types of data (like json, image and etc)
       String result = stmtInfo.output.data.plain_text;
-      
+
       // check table magic result first
       if (stmtInfo.output.data.application_livy_table_json != null) {
         StringBuilder outputBuilder = new StringBuilder();
         boolean notFirstColumn = false;
-        
+
         for (Map header : stmtInfo.output.data.application_livy_table_json.headers) {
           if (notFirstColumn) {
             outputBuilder.append("\t");
@@ -325,17 +345,17 @@ public abstract class BaseLivyInterprereter extends Interpreter {
           outputBuilder.append(header.get("name"));
           notFirstColumn = true;
         }
-        
+
         outputBuilder.append("\n");
         for (List<Object> row : stmtInfo.output.data.application_livy_table_json.records) {
           outputBuilder.append(StringUtils.join(row, "\t"));
-          outputBuilder.append("\n");          
-        }        
+          outputBuilder.append("\n");
+        }
         return new InterpreterResult(InterpreterResult.Code.SUCCESS,
-          InterpreterResult.Type.TABLE, outputBuilder.toString());
-      } else if (stmtInfo.output.data.image_png != null) {        
+            InterpreterResult.Type.TABLE, outputBuilder.toString());
+      } else if (stmtInfo.output.data.image_png != null) {
         return new InterpreterResult(InterpreterResult.Code.SUCCESS,
-          InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png);
+            InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png);
       } else if (result != null) {
         result = result.trim();
         if (result.startsWith("<link")
@@ -378,13 +398,56 @@ public abstract class BaseLivyInterprereter extends Interpreter {
     callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId + "/cancel", "POST");
   }
 
-  private RestTemplate getRestTemplate() {
+
+  private RestTemplate createRestTemplate() {
+    HttpClient httpClient = null;
+    if (livyURL.startsWith("https:")) {
+      String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
+      String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword");
+      if (StringUtils.isBlank(keystoreFile)) {
+        throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl");
+      }
+      if (StringUtils.isBlank(password)) {
+        throw new RuntimeException("No zeppelin.livy.ssl.trustStorePassword specified " +
+            "for livy ssl");
+      }
+      FileInputStream inputStream = null;
+      try {
+        inputStream = new FileInputStream(keystoreFile);
+        KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        trustStore.load(new FileInputStream(keystoreFile), password.toCharArray());
+        SSLContext sslContext = SSLContexts.custom()
+            .loadTrustMaterial(trustStore)
+            .build();
+        SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
+        httpClient = HttpClients.custom().setSSLSocketFactory(csf).build();
+      } catch (Exception e) {
+        throw new RuntimeException("Failed to create SSL HttpClient", e);
+      } finally {
+        if (inputStream != null) {
+          try {
+            inputStream.close();
+          } catch (IOException e) {
+            LOGGER.error("Failed to close keystore file", e);
+          }
+        }
+      }
+    }
+
     String keytabLocation = property.getProperty("zeppelin.livy.keytab");
     String principal = property.getProperty("zeppelin.livy.principal");
     if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
-      return new KerberosRestTemplate(keytabLocation, principal);
+      if (httpClient == null) {
+        return new KerberosRestTemplate(keytabLocation, principal);
+      } else {
+        return new KerberosRestTemplate(keytabLocation, principal, httpClient);
+      }
+    }
+    if (httpClient == null) {
+      return new RestTemplate();
+    } else {
+      return new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient));
     }
-    return new RestTemplate();
   }
 
   private String callRestAPI(String targetURL, String method) throws LivyException {
@@ -395,7 +458,6 @@ public abstract class BaseLivyInterprereter extends Interpreter {
       throws LivyException {
     targetURL = livyURL + targetURL;
     LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData);
-    RestTemplate restTemplate = getRestTemplate();
     HttpHeaders headers = new HttpHeaders();
     headers.add("Content-Type", "application/json");
     headers.add("X-Requested-By", "zeppelin");
@@ -537,6 +599,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
   private static class StatementInfo {
     public Integer id;
     public String state;
+    public double progress;
     public StatementOutput output;
 
     public StatementInfo() {
@@ -581,11 +644,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
         @SerializedName("application/vnd.livy.table.v1+json")
         public TableMagic application_livy_table_json;
       }
-      
+
       private static class TableMagic {
         @SerializedName("headers")
         List<Map> headers;
-        
+
         @SerializedName("data")
         List<List> records;
       }

livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java

@@ -28,6 +28,7 @@ public class LivyVersion {
 
   protected static final LivyVersion LIVY_0_2_0 = LivyVersion.fromVersionString("0.2.0");
   protected static final LivyVersion LIVY_0_3_0 = LivyVersion.fromVersionString("0.3.0");
+  protected static final LivyVersion LIVY_0_4_0 = LivyVersion.fromVersionString("0.4.0");
 
   private int version;
   private String versionString;
@@ -74,6 +75,10 @@ public class LivyVersion {
     return this.newerThanEquals(LIVY_0_3_0);
   }
 
+  public boolean isGetProgressSupported() {
+    return this.newerThanEquals(LIVY_0_4_0);
+  }
+
   public boolean equals(Object versionToCompare) {
     return version == ((LivyVersion) versionToCompare).version;
   }

livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java

@@ -36,6 +36,7 @@ public class LivySQLInterpreterTest {
   @Before
   public void setUp() {
     Properties properties = new Properties();
+    properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
     properties.setProperty("zeppelin.livy.session.create_timeout", "120");
     properties.setProperty("zeppelin.livy.spark.sql.maxResult", "3");
     sqlInterpreter = new LivySparkSQLInterpreter(properties);

notebook/2A94M5J1Z/note.json

@@ -32,9 +32,6 @@
         "params": {},
         "forms": {}
       },
-      "apps": [],
-      "jobName": "paragraph_1423836981412_-1007008116",
-      "id": "20150213-231621_168813393",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -44,6 +41,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423836981412_-1007008116",
+      "id": "20150213-231621_168813393",
       "dateCreated": "Feb 13, 2015 11:16:21 PM",
       "dateStarted": "Dec 17, 2016 3:32:15 PM",
       "dateFinished": "Dec 17, 2016 3:32:18 PM",
@@ -78,9 +78,6 @@
         "params": {},
         "forms": {}
       },
-      "apps": [],
-      "jobName": "paragraph_1423500779206_-1502780787",
-      "id": "20150210-015259_1403135953",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -90,6 +87,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423500779206_-1502780787",
+      "id": "20150210-015259_1403135953",
       "dateCreated": "Feb 10, 2015 1:52:59 AM",
       "dateStarted": "Dec 17, 2016 3:30:09 PM",
       "dateFinished": "Dec 17, 2016 3:30:58 PM",
@@ -99,16 +99,17 @@
     {
       "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
       "user": "anonymous",
-      "dateUpdated": "Dec 17, 2016 3:30:13 PM",
+      "dateUpdated": "Mar 17, 2017 12:18:02 PM",
       "config": {
         "colWidth": 4.0,
         "results": [
           {
             "graph": {
-              "mode": "table",
-              "height": 300.0,
+              "mode": "multiBarChart",
+              "height": 366.0,
               "optionOpen": false
-            }
+            },
+            "helium": {}
           }
         ],
         "enabled": true,
@@ -122,9 +123,6 @@
         "params": {},
         "forms": {}
       },
-      "apps": [],
-      "jobName": "paragraph_1423500782552_-1439281894",
-      "id": "20150210-015302_1492795503",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -134,6 +132,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423500782552_-1439281894",
+      "id": "20150210-015302_1492795503",
       "dateCreated": "Feb 10, 2015 1:53:02 AM",
       "dateStarted": "Dec 17, 2016 3:30:13 PM",
       "dateFinished": "Dec 17, 2016 3:31:04 PM",
@@ -143,16 +144,17 @@
     {
       "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
       "user": "anonymous",
-      "dateUpdated": "Dec 17, 2016 3:30:16 PM",
+      "dateUpdated": "Mar 17, 2017 12:17:39 PM",
       "config": {
         "colWidth": 4.0,
         "results": [
           {
             "graph": {
-              "mode": "table",
-              "height": 300.0,
+              "mode": "multiBarChart",
+              "height": 294.0,
               "optionOpen": false
-            }
+            },
+            "helium": {}
           }
         ],
         "enabled": true,
@@ -174,9 +176,6 @@
           }
         }
       },
-      "apps": [],
-      "jobName": "paragraph_1423720444030_-1424110477",
-      "id": "20150212-145404_867439529",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -186,6 +185,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423720444030_-1424110477",
+      "id": "20150212-145404_867439529",
       "dateCreated": "Feb 12, 2015 2:54:04 PM",
       "dateStarted": "Dec 17, 2016 3:30:58 PM",
       "dateFinished": "Dec 17, 2016 3:31:07 PM",
@@ -195,16 +197,17 @@
     {
       "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
       "user": "anonymous",
-      "dateUpdated": "Dec 17, 2016 3:30:18 PM",
+      "dateUpdated": "Mar 17, 2017 12:18:18 PM",
       "config": {
         "colWidth": 4.0,
         "results": [
           {
             "graph": {
-              "mode": "table",
-              "height": 300.0,
+              "mode": "stackedAreaChart",
+              "height": 280.0,
               "optionOpen": false
-            }
+            },
+            "helium": {}
           }
         ],
         "enabled": true,
@@ -237,9 +240,6 @@
           }
         }
       },
-      "apps": [],
-      "jobName": "paragraph_1423836262027_-210588283",
-      "id": "20150213-230422_1600658137",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -249,6 +249,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423836262027_-210588283",
+      "id": "20150213-230422_1600658137",
       "dateCreated": "Feb 13, 2015 11:04:22 PM",
       "dateStarted": "Dec 17, 2016 3:31:05 PM",
       "dateFinished": "Dec 17, 2016 3:31:09 PM",
@@ -283,9 +286,6 @@
         "params": {},
         "forms": {}
       },
-      "apps": [],
-      "jobName": "paragraph_1423836268492_216498320",
-      "id": "20150213-230428_1231780373",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -295,6 +295,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1423836268492_216498320",
+      "id": "20150213-230428_1231780373",
       "dateCreated": "Feb 13, 2015 11:04:28 PM",
       "dateStarted": "Dec 17, 2016 3:30:24 PM",
       "dateFinished": "Dec 17, 2016 3:30:29 PM",
@@ -329,9 +332,6 @@
         "params": {},
         "forms": {}
       },
-      "apps": [],
-      "jobName": "paragraph_1427420818407_872443482",
-      "id": "20150326-214658_12335843",
       "results": {
         "code": "SUCCESS",
         "msg": [
@@ -341,6 +341,9 @@
           }
         ]
       },
+      "apps": [],
+      "jobName": "paragraph_1427420818407_872443482",
+      "id": "20150326-214658_12335843",
       "dateCreated": "Mar 26, 2015 9:46:58 PM",
       "dateStarted": "Dec 17, 2016 3:30:34 PM",
       "dateFinished": "Dec 17, 2016 3:30:34 PM",
@@ -364,25 +367,7 @@
   "name": "Zeppelin Tutorial/Basic Features (Spark)",
   "id": "2A94M5J1Z",
   "angularObjects": {
-    "2C6WUGPNH:shared_process": [],
-    "2C4A8RJNB:shared_process": [],
-    "2C4DTK2ZT:shared_process": [],
-    "2C6XKJWBR:shared_process": [],
-    "2C6AHZPMK:shared_process": [],
-    "2C5SU66WQ:shared_process": [],
-    "2C6AMJ98Q:shared_process": [],
-    "2C4AJZK72:shared_process": [],
-    "2C3STPSD7:shared_process": [],
-    "2C4FJN9CK:shared_process": [],
-    "2C3CW6JBY:shared_process": [],
-    "2C5UPQX6Q:shared_process": [],
-    "2C5873KN4:shared_process": [],
-    "2C5719XN4:shared_process": [],
-    "2C52DE5G3:shared_process": [],
-    "2C4G28E63:shared_process": [],
-    "2C6CU96BC:shared_process": [],
-    "2C49A6WY3:shared_process": [],
-    "2C3NE73HG:shared_process": []
+    "2C73DY9P9:shared_process": []
   },
   "config": {
     "looknfeel": "default"

python/pom.xml

@@ -98,6 +98,43 @@
         </executions>
       </plugin>
 
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>wagon-maven-plugin</artifactId>
+        <version>1.0</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals><goal>download-single</goal></goals>
+            <configuration>
+              <url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
+              <fromFile>py4j-${py4j.version}.zip</fromFile>
+              <toFile>${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip</toFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <version>1.7</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <configuration>
+              <target>
+                <unzip src="${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip"
+                       dest="${project.build.directory}/../../interpreter/python"/>
+              </target>
+            </configuration>
+            <goals>
+              <goal>run</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>

python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java

@@ -21,10 +21,8 @@ import org.apache.zeppelin.scheduler.Scheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.*;
+import java.nio.file.Paths;
 import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -37,6 +35,7 @@ public class PythonDockerInterpreter extends Interpreter {
   Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
   Pattern deactivatePattern = Pattern.compile("deactivate");
   Pattern helpPattern = Pattern.compile("help");
+  private File zeppelinHome;
 
   public PythonDockerInterpreter(Properties property) {
     super(property);
@@ -44,7 +43,11 @@ public class PythonDockerInterpreter extends Interpreter {
 
   @Override
   public void open() {
-
+    if (System.getenv("ZEPPELIN_HOME") != null) {
+      zeppelinHome = new File(System.getenv("ZEPPELIN_HOME"));
+    } else {
+      zeppelinHome = Paths.get("..").toAbsolutePath().toFile();
+    }
   }
 
   @Override
@@ -54,6 +57,7 @@ public class PythonDockerInterpreter extends Interpreter {
 
   @Override
   public InterpreterResult interpret(String st, InterpreterContext context) {
+    File pythonScript = new File(getPythonInterpreter().getScriptPath());
     InterpreterOutput out = context.out;
 
     Matcher activateMatcher = activatePattern.matcher(st);
@@ -66,7 +70,28 @@ public class PythonDockerInterpreter extends Interpreter {
     } else if (activateMatcher.matches()) {
       String image = activateMatcher.group(1);
       pull(out, image);
-      setPythonCommand("docker run -i --rm " + image + " python -iu");
+
+      // mount pythonscript dir
+      String mountPythonScript = "-v " +
+          pythonScript.getParentFile().getAbsolutePath() +
+          ":/_zeppelin_tmp ";
+
+      // mount zeppelin dir
+      String mountPy4j = "-v " +
+          zeppelinHome.getAbsolutePath() +
+          ":/_zeppelin ";
+
+      // set PYTHONPATH
+      String pythonPath = ":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PY4JPATH + ":" +
+          ":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PYTHON_LIBS;
+
+      setPythonCommand("docker run -i --rm " +
+          mountPythonScript +
+          mountPy4j +
+          "-e PYTHONPATH=\"" + pythonPath + "\" " +
+          image + " " +
+          getPythonInterpreter().getPythonBindPath() + " " +
+          "/_zeppelin_tmp/" + pythonScript.getName());
       restartPythonProcess();
       out.clear();
       return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated");
@@ -79,6 +104,7 @@ public class PythonDockerInterpreter extends Interpreter {
     }
   }
 
+
   public void setPythonCommand(String cmd) {
     PythonInterpreter python = getPythonInterpreter();
     python.setPythonCommand(cmd);

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -18,20 +18,38 @@
 package org.apache.zeppelin.python;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.net.ServerSocket;
+import java.io.OutputStreamWriter;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.net.*;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.exec.ExecuteResultHandler;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
 import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -39,32 +57,164 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import py4j.GatewayServer;
+import py4j.commands.Command;
 
 /**
  * Python interpreter for Zeppelin.
  */
-public class PythonInterpreter extends Interpreter {
+public class PythonInterpreter extends Interpreter implements ExecuteResultHandler {
   private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreter.class);
-
-  public static final String BOOTSTRAP_PY = "/bootstrap.py";
-  public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py";
-  public static final String ZEPPELIN_PYTHON = "zeppelin.python";
+  public static final String ZEPPELIN_PYTHON = "python/zeppelin_python.py";
+  public static final String ZEPPELIN_PY4JPATH = "interpreter/python/py4j-0.9.2/src";
+  public static final String ZEPPELIN_PYTHON_LIBS = "interpreter/lib/python";
   public static final String DEFAULT_ZEPPELIN_PYTHON = "python";
   public static final String MAX_RESULT = "zeppelin.python.maxResult";
 
-  private Integer port;
-  private GatewayServer gatewayServer;
-  private Boolean py4JisInstalled = false;
   private InterpreterContext context;
   private Pattern errorInLastLine = Pattern.compile(".*(Error|Exception): .*$");
   private String pythonPath;
   private int maxResult;
+  private String py4jLibPath;
+  private String pythonLibPath;
 
-  PythonProcess process = null;
-  private String pythonCommand = null;
+  private String pythonCommand;
+
+  private GatewayServer gatewayServer;
+  private DefaultExecutor executor;
+  private int port;
+  private InterpreterOutputStream outputStream;
+  private BufferedWriter ins;
+  private PipedInputStream in;
+  private ByteArrayOutputStream input;
+  private String scriptPath;
+  boolean pythonscriptRunning = false;
+  private static final int MAX_TIMEOUT_SEC = 10;
+
+  private long pythonPid = 0;
+
+  Integer statementSetNotifier = new Integer(0);
 
   public PythonInterpreter(Properties property) {
     super(property);
+    try {
+      File scriptFile = File.createTempFile("zeppelin_python-", ".py", new File("/tmp"));
+      scriptPath = scriptFile.getAbsolutePath();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
+  }
+
+  private String workingDir() {
+    URL myURL = getClass().getProtectionDomain().getCodeSource().getLocation();
+    java.net.URI myURI = null;
+    try {
+      myURI = myURL.toURI();
+    } catch (URISyntaxException e1)
+    {}
+    String path = java.nio.file.Paths.get(myURI).toFile().toString();
+    return path;
+  }
+
+  private void createPythonScript() {
+    File out = new File(scriptPath);
+
+    if (out.exists() && out.isDirectory()) {
+      throw new InterpreterException("Can't create python script " + out.getAbsolutePath());
+    }
+
+    copyFile(out, ZEPPELIN_PYTHON);
+    logger.info("File {} created", scriptPath);
+  }
+
+  public String getScriptPath() {
+    return scriptPath;
+  }
+
+  private void copyFile(File out, String sourceFile) {
+    ClassLoader classLoader = getClass().getClassLoader();
+    try {
+      FileOutputStream outStream = new FileOutputStream(out);
+      IOUtils.copy(
+          classLoader.getResourceAsStream(sourceFile),
+          outStream);
+      outStream.close();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
+  }
+
+  private void createGatewayServerAndStartScript() throws UnknownHostException {
+    createPythonScript();
+    if (System.getenv("ZEPPELIN_HOME") != null) {
+      py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
+      pythonLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PYTHON_LIBS;
+    } else {
+      Path workingPath = Paths.get("..").toAbsolutePath();
+      py4jLibPath = workingPath + File.separator + ZEPPELIN_PY4JPATH;
+      pythonLibPath = workingPath + File.separator + ZEPPELIN_PYTHON_LIBS;
+    }
+
+    port = findRandomOpenPortOnAllLocalInterfaces();
+    gatewayServer = new GatewayServer(this,
+        port,
+        GatewayServer.DEFAULT_PYTHON_PORT,
+        InetAddress.getByName("0.0.0.0"),
+        InetAddress.getByName("0.0.0.0"),
+        GatewayServer.DEFAULT_CONNECT_TIMEOUT,
+        GatewayServer.DEFAULT_READ_TIMEOUT,
+        (List) null);
+
+    gatewayServer.start();
+
+    // Run python shell
+    String pythonCmd = getPythonCommand();
+    CommandLine cmd = CommandLine.parse(pythonCmd);
+
+    if (!pythonCmd.endsWith(".py")) {
+      // PythonDockerInterpreter set pythoncmd with script
+      cmd.addArgument(getScriptPath(), false);
+    }
+    cmd.addArgument(Integer.toString(port), false);
+    cmd.addArgument(getLocalIp(), false);
+
+    executor = new DefaultExecutor();
+    outputStream = new InterpreterOutputStream(logger);
+    PipedOutputStream ps = new PipedOutputStream();
+    in = null;
+    try {
+      in = new PipedInputStream(ps);
+    } catch (IOException e1) {
+      throw new InterpreterException(e1);
+    }
+    ins = new BufferedWriter(new OutputStreamWriter(ps));
+    input = new ByteArrayOutputStream();
+
+    PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
+    executor.setStreamHandler(streamHandler);
+    executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
+
+    try {
+      Map env = EnvironmentUtils.getProcEnvironment();
+      if (!env.containsKey("PYTHONPATH")) {
+        env.put("PYTHONPATH", py4jLibPath + File.pathSeparator + pythonLibPath);
+      } else {
+        env.put("PYTHONPATH", env.get("PYTHONPATH") + File.pathSeparator +
+                py4jLibPath + File.pathSeparator + pythonLibPath);
+      }
+
+      logger.info("cmd = {}", cmd.toString());
+      executor.execute(cmd, env, this);
+      pythonscriptRunning = true;
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
+
+    try {
+      input.write("import sys, getopt\n".getBytes());
+      ins.flush();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
   }
 
   @Override
@@ -72,66 +222,99 @@ public class PythonInterpreter extends Interpreter {
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
-      registerHook(HookType.POST_EXEC_DEV, "\nz._displayhook()");
+      registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
     }
-
-    // Add zeppelin-bundled libs to PYTHONPATH
-    setPythonPath("../interpreter/lib/python:$PYTHONPATH");
-    LOG.info("Starting Python interpreter ---->");
-    LOG.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
-
-    maxResult = Integer.valueOf(getProperty(MAX_RESULT));
-    process = getPythonProcess();
-
+    // Add matplotlib display hook
     try {
-      process.open();
-    } catch (IOException e) {
-      LOG.error("Can't start the python process", e);
-    }
-
-    try {
-      LOG.info("python PID : " + process.getPid());
-    } catch (Exception e) {
-      LOG.warn("Can't find python pid process", e);
-    }
-
-    try {
-      LOG.info("Bootstrap interpreter with " + BOOTSTRAP_PY);
-      bootStrapInterpreter(BOOTSTRAP_PY);
-    } catch (IOException e) {
-      LOG.error("Can't execute " + BOOTSTRAP_PY + " to initiate python process", e);
-    }
-
-    py4JisInstalled = isPy4jInstalled();
-    if (py4JisInstalled) {
-      port = findRandomOpenPortOnAllLocalInterfaces();
-      LOG.info("Py4j gateway port : " + port);
-      try {
-        gatewayServer = new GatewayServer(this, port);
-        gatewayServer.start();
-        LOG.info("Bootstrap inputs with " + BOOTSTRAP_INPUT_PY);
-        bootStrapInterpreter(BOOTSTRAP_INPUT_PY);
-      } catch (IOException e) {
-        LOG.error("Can't execute " + BOOTSTRAP_INPUT_PY + " to " +
-            "initialize Zeppelin inputs in python process", e);
-      }
+      createGatewayServerAndStartScript();
+    } catch (UnknownHostException e) {
+      throw new InterpreterException(e);
     }
   }
 
   @Override
   public void close() {
-    LOG.info("closing Python interpreter <----");
+    pythonscriptRunning = false;
+    pythonScriptInitialized = false;
+
     try {
-      if (process != null) {
-        process.close();
-        process = null;
-      }
-      if (gatewayServer != null) {
-        gatewayServer.shutdown();
-      }
+      ins.flush();
+      ins.close();
+      input.flush();
+      input.close();
     } catch (IOException e) {
-      LOG.error("Can't close the interpreter", e);
+      e.printStackTrace();
     }
+
+    executor.getWatchdog().destroyProcess();
+    new File(scriptPath).delete();
+    gatewayServer.shutdown();
+
+    // wait until getStatements stop
+    synchronized (statementSetNotifier) {
+      try {
+        statementSetNotifier.wait(1500);
+      } catch (InterruptedException e) {
+      }
+      statementSetNotifier.notify();
+    }
+  }
+
+  PythonInterpretRequest pythonInterpretRequest = null;
+  /**
+   * Result class of python interpreter
+   */
+  public class PythonInterpretRequest {
+    public String statements;
+
+    public PythonInterpretRequest(String statements) {
+      this.statements = statements;
+    }
+
+    public String statements() {
+      return statements;
+    }
+  }
+
+  public PythonInterpretRequest getStatements() {
+    synchronized (statementSetNotifier) {
+      while (pythonInterpretRequest == null && pythonscriptRunning && pythonScriptInitialized) {
+        try {
+          statementSetNotifier.wait(1000);
+        } catch (InterruptedException e) {
+        }
+      }
+      PythonInterpretRequest req = pythonInterpretRequest;
+      pythonInterpretRequest = null;
+      return req;
+    }
+  }
+
+  String statementOutput = null;
+  boolean statementError = false;
+  Integer statementFinishedNotifier = new Integer(0);
+
+  public void setStatementsFinished(String out, boolean error) {
+    synchronized (statementFinishedNotifier) {
+      statementOutput = out;
+      statementError = error;
+      statementFinishedNotifier.notify();
+    }
+  }
+
+  boolean pythonScriptInitialized = false;
+  Integer pythonScriptInitializeNotifier = new Integer(0);
+
+  public void onPythonScriptInitialized(long pid) {
+    pythonPid = pid;
+    synchronized (pythonScriptInitializeNotifier) {
+      pythonScriptInitialized = true;
+      pythonScriptInitializeNotifier.notifyAll();
+    }
+  }
+
+  public void appendOutput(String message) throws IOException {
+    outputStream.getInterpreterOutput().write(message);
   }
 
   @Override
@@ -139,44 +322,99 @@ public class PythonInterpreter extends Interpreter {
     if (cmd == null || cmd.isEmpty()) {
       return new InterpreterResult(Code.SUCCESS, "");
     }
+
     this.context = contextInterpreter;
-    String output = sendCommandToPython(cmd);
 
-    InterpreterResult result;
-    if (pythonErrorIn(output)) {
-      result = new InterpreterResult(Code.ERROR, output.replaceAll("\\.\\.\\.", ""));
-    } else {
-      result = new InterpreterResult(Code.SUCCESS, output);
+    if (!pythonscriptRunning) {
+      return new InterpreterResult(Code.ERROR, "python process not running"
+        + outputStream.toString());
     }
-    return result;
-  }
 
-  /**
-   * Checks if there is a syntax error or an exception
-   *
-   * @param output Python interpreter output
-   * @return true if syntax error or exception has happened
-   */
-  private boolean pythonErrorIn(String output) {
-    boolean isError = false;
-    String[] outputMultiline = output.split("\n");
-    Matcher errorMatcher;
-    for (String row : outputMultiline) {
-      errorMatcher = errorInLastLine.matcher(row);
-      if (errorMatcher.find() == true) {
-        isError = true;
-        break;
+    outputStream.setInterpreterOutput(context.out);
+
+    synchronized (pythonScriptInitializeNotifier) {
+      long startTime = System.currentTimeMillis();
+      while (pythonScriptInitialized == false
+        && pythonscriptRunning
+        && System.currentTimeMillis() - startTime < MAX_TIMEOUT_SEC * 1000) {
+        try {
+          pythonScriptInitializeNotifier.wait(1000);
+        } catch (InterruptedException e) {
+        }
       }
     }
-    return isError;
+
+    List<InterpreterResultMessage> errorMessage;
+    try {
+      context.out.flush();
+      errorMessage = context.out.toInterpreterResultMessage();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
+
+    if (pythonscriptRunning == false) {
+      // python script failed to initialize and terminated
+      errorMessage.add(new InterpreterResultMessage(
+        InterpreterResult.Type.TEXT, "failed to start python"));
+      return new InterpreterResult(Code.ERROR, errorMessage);
+    }
+    if (pythonScriptInitialized == false) {
+      // timeout. didn't get initialized message
+      errorMessage.add(new InterpreterResultMessage(
+        InterpreterResult.Type.TEXT, "python is not responding"));
+      return new InterpreterResult(Code.ERROR, errorMessage);
+    }
+
+    pythonInterpretRequest = new PythonInterpretRequest(cmd);
+    statementOutput = null;
+
+    synchronized (statementSetNotifier) {
+      statementSetNotifier.notify();
+    }
+
+    synchronized (statementFinishedNotifier) {
+      while (statementOutput == null) {
+        try {
+          statementFinishedNotifier.wait(1000);
+        } catch (InterruptedException e) {
+        }
+      }
+    }
+
+    if (statementError) {
+      return new InterpreterResult(Code.ERROR, statementOutput);
+    } else {
+
+      try {
+        context.out.flush();
+      } catch (IOException e) {
+        throw new InterpreterException(e);
+      }
+
+      return new InterpreterResult(Code.SUCCESS);
+    }
+  }
+
+  public InterpreterContext getCurrentInterpreterContext() {
+    return context;
+  }
+
+  public void interrupt() throws IOException {
+    if (pythonPid > -1) {
+      logger.info("Sending SIGINT signal to PID : " + pythonPid);
+      Runtime.getRuntime().exec("kill -SIGINT " + pythonPid);
+    } else {
+      logger.warn("Non UNIX/Linux system, close the interpreter");
+      close();
+    }
   }
 
   @Override
   public void cancel(InterpreterContext context) {
     try {
-      process.interrupt();
+      interrupt();
     } catch (IOException e) {
-      LOG.error("Can't interrupt the python interpreter", e);
+      e.printStackTrace();
     }
   }
 
@@ -201,28 +439,26 @@ public class PythonInterpreter extends Interpreter {
     return null;
   }
 
-  public void setPythonPath(String pythonPath) {
-    this.pythonPath = pythonPath;
-  }
-
-  public PythonProcess getPythonProcess() {
-    if (process == null) {
-      String binPath = getProperty(ZEPPELIN_PYTHON);
-      if (pythonCommand != null) {
-        binPath = pythonCommand;
-      }
-      return new PythonProcess(binPath, pythonPath);
-    } else {
-      return process;
-    }
-  }
-
   public void setPythonCommand(String cmd) {
+    logger.info("Set Python Command : {}", cmd);
     pythonCommand = cmd;
   }
 
-  public String getPythonCommand() {
-    return pythonCommand;
+  private String getPythonCommand() {
+    if (pythonCommand == null) {
+      return getPythonBindPath();
+    } else {
+      return pythonCommand;
+    }
+  }
+
+  public String getPythonBindPath() {
+    String path = getProperty("zeppelin.python");
+    if (path == null) {
+      return DEFAULT_ZEPPELIN_PYTHON;
+    } else {
+      return path;
+    }
   }
 
   private Job getRunningJob(String paragraphId) {
@@ -237,24 +473,6 @@ public class PythonInterpreter extends Interpreter {
     return foundJob;
   }
 
-
-  /**
-   * Sends given text to Python interpreter, blocks and returns the output
-   * @param cmd Python expression text
-   * @return output
-   */
-  String sendCommandToPython(String cmd) {
-    String output = "";
-    LOG.debug("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 200) + "..." : cmd));
-    try {
-      output = process.sendAndGetResult(cmd);
-    } catch (IOException e) {
-      LOG.error("Error when sending commands to python process", e);
-    }
-    LOG.debug("Got : \n" + output);
-    return output;
-  }
-
   void bootStrapInterpreter(String file) throws IOException {
     BufferedReader bootstrapReader = new BufferedReader(
         new InputStreamReader(
@@ -265,24 +483,22 @@ public class PythonInterpreter extends Interpreter {
     while ((line = bootstrapReader.readLine()) != null) {
       bootstrapCode += line + "\n";
     }
-    if (py4JisInstalled && port != null && port != -1) {
-      bootstrapCode = bootstrapCode.replaceAll("\\%PORT\\%", port.toString());
-    }
-    LOG.info("Bootstrap python interpreter with code from \n " + file);
-    sendCommandToPython(bootstrapCode);
+
+    interpret(bootstrapCode, context);
   }
 
   public GUI getGui() {
     return context.getGui();
   }
 
-  public Integer getPy4jPort() {
-    return port;
-  }
-
-  public Boolean isPy4jInstalled() {
-    String output = sendCommandToPython("\n\nimport py4j\n");
-    return !output.contains("ImportError");
+  String getLocalIp() {
+    try {
+      return Inet4Address.getLocalHost().getHostAddress();
+    } catch (UnknownHostException e) {
+      logger.error("can't get local IP", e);
+    }
+    // fall back to loopback addreess
+    return "127.0.0.1";
   }
 
   private int findRandomOpenPortOnAllLocalInterfaces() {
@@ -299,4 +515,16 @@ public class PythonInterpreter extends Interpreter {
   public int getMaxResult() {
     return maxResult;
   }
+
+  @Override
+  public void onProcessComplete(int exitValue) {
+    pythonscriptRunning = false;
+    logger.info("python process terminated. exit code " + exitValue);
+  }
+
+  @Override
+  public void onProcessFailed(ExecuteException e) {
+    pythonscriptRunning = false;
+    logger.error("python process failed", e);
+  }
 }

python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java

@@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory;
 public class PythonInterpreterPandasSql extends Interpreter {
   private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreterPandasSql.class);
 
-  private String SQL_BOOTSTRAP_FILE_PY = "/bootstrap_sql.py";
+  private String SQL_BOOTSTRAP_FILE_PY = "/python/bootstrap_sql.py";
 
   public PythonInterpreterPandasSql(Properties property) {
     super(property);
@@ -64,25 +64,17 @@ public class PythonInterpreterPandasSql extends Interpreter {
   @Override
   public void open() {
     LOG.info("Open Python SQL interpreter instance: {}", this.toString());
+
     try {
       LOG.info("Bootstrap {} interpreter with {}", this.toString(), SQL_BOOTSTRAP_FILE_PY);
       PythonInterpreter python = getPythonInterpreter();
+
       python.bootStrapInterpreter(SQL_BOOTSTRAP_FILE_PY);
     } catch (IOException e) {
       LOG.error("Can't execute " + SQL_BOOTSTRAP_FILE_PY + " to import SQL dependencies", e);
     }
   }
 
-  /**
-   * Checks if Python dependencies pandas and pandasql are installed
-   * @return True if they are
-   */
-  boolean isPandasAndPandasqlInstalled() {
-    PythonInterpreter python = getPythonInterpreter();
-    String output = python.sendCommandToPython("\n\nimport pandas\nimport pandasql\n");
-    return !output.contains("ImportError");
-  }
-
   @Override
   public void close() {
     LOG.info("Close Python SQL interpreter instance: {}", this.toString());
@@ -94,7 +86,8 @@ public class PythonInterpreterPandasSql extends Interpreter {
   public InterpreterResult interpret(String st, InterpreterContext context) {
     LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
     Interpreter python = getPythonInterpreter();
-    return python.interpret("z.show(pysqldf('" + st + "'))", context);
+
+    return python.interpret("z.show(pysqldf('" + st + "'))\nz._displayhook()", context);
   }
 
   @Override

python/src/main/java/org/apache/zeppelin/python/PythonProcess.java

@@ -1,138 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.zeppelin.python;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.OutputStream;
-import java.lang.reflect.Field;
-
-/**
- * Object encapsulated interactive
- * Python process (REPL) used by python interpreter
- */
-public class PythonProcess {
-  private static final Logger logger = LoggerFactory.getLogger(PythonProcess.class);
-  private static final String STATEMENT_END = "*!?flush reader!?*";
-  InputStream stdout;
-  OutputStream stdin;
-  PrintWriter writer;
-  BufferedReader reader;
-  Process process;
-
-  private String binPath;
-  private String pythonPath;
-  private long pid;
-
-  public PythonProcess(String binPath, String pythonPath) {
-    this.binPath = binPath;
-    this.pythonPath = pythonPath;
-  }
-
-  public void open() throws IOException {
-    ProcessBuilder builder;
-    boolean hasParams = binPath.split(" ").length > 1;
-    if (System.getProperty("os.name").toLowerCase().contains("windows")) {
-      if (hasParams) {
-        builder = new ProcessBuilder(binPath.split(" "));
-      } else {
-        builder = new ProcessBuilder(binPath, "-iu");
-      }
-    } else {
-      String cmd;
-      if (hasParams) {
-        cmd = binPath;
-      } else {
-        cmd = binPath + " -iu";
-      }
-      builder = new ProcessBuilder("bash", "-c", cmd);
-      if (pythonPath != null) {
-        builder.environment().put("PYTHONPATH", pythonPath);
-      }
-    }
-
-    builder.redirectErrorStream(true);
-    process = builder.start();
-    stdout = process.getInputStream();
-    stdin = process.getOutputStream();
-    writer = new PrintWriter(stdin, true);
-    reader = new BufferedReader(new InputStreamReader(stdout));
-    try {
-      pid = findPid();
-    } catch (Exception e) {
-      logger.warn("Can't find python pid process", e);
-      pid = -1;
-    }
-  }
-
-  public void close() throws IOException {
-    process.destroy();
-    reader.close();
-    writer.close();
-    stdin.close();
-    stdout.close();
-  }
-
-  public void interrupt() throws IOException {
-    if (pid > -1) {
-      logger.info("Sending SIGINT signal to PID : " + pid);
-      Runtime.getRuntime().exec("kill -SIGINT " + pid);
-    } else {
-      logger.warn("Non UNIX/Linux system, close the interpreter");
-      close();
-    }
-  }
-
-  public String sendAndGetResult(String cmd) throws IOException {
-    writer.println(cmd);
-    writer.println();
-    writer.println("\"" + STATEMENT_END + "\"");
-    StringBuilder output = new StringBuilder();
-    String line = null;
-
-    while ((line = reader.readLine()) != null &&
-        !line.contains(STATEMENT_END)) {
-      logger.debug("Read line from python shell : " + line);
-      output.append(line + "\n");
-    }
-
-    return output.toString();
-  }
-
-  private long findPid() throws NoSuchFieldException, IllegalAccessException {
-    long pid = -1;
-    if (process.getClass().getName().equals("java.lang.UNIXProcess")) {
-      Field f = process.getClass().getDeclaredField("pid");
-      f.setAccessible(true);
-      pid = f.getLong(process);
-      f.setAccessible(false);
-    }
-    return pid;
-  }
-
-  public long getPid() {
-    return pid;
-  }
-
-}

python/src/main/resources/__init__.py

@@ -1,14 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

python/src/main/resources/bootstrap.py

@@ -1,234 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# PYTHON 2 / 3 compatibility :
-# bootstrap.py must be runnable with Python 2 or 3
-
-import os
-import sys
-import signal
-import base64
-import warnings
-from io import BytesIO
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
-
-def intHandler(signum, frame):  # Set the signal handler
-    print ("Paragraph interrupted")
-    raise KeyboardInterrupt()
-
-signal.signal(signal.SIGINT, intHandler)
-# set prompt as empty string so that java side don't need to remove the prompt.
-sys.ps1=""
-
-def help():
-    print("""%html
- <h2>Python Interpreter help</h2>
-
- <h3>Python 2 & 3 compatibility</h3>
- <p>The interpreter is compatible with Python 2 & 3.<br/>
- To change Python version,
- change in the interpreter configuration the python to the
- desired version (example : python=/usr/bin/python3)</p>
-
- <h3>Python modules</h3>
- <p>The interpreter can use all modules already installed
- (with pip, easy_install, etc)</p>
-
- <h3>Forms</h3>
- You must install py4j in order to use
- the form feature (pip install py4j)
- <h4>Input form</h4>
- <pre>print (z.input("f1","defaultValue"))</pre>
- <h4>Selection form</h4>
- <pre>print(z.select("f2", [("o1","1"), ("o2","2")],2))</pre>
- <h4>Checkbox form</h4>
- <pre> print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))</pre>')
-
- <h3>Matplotlib graph</h3>
- <div>The interpreter can display matplotlib graph with
- the function z.show()</div>
- <div> You need to already have matplotlib module installed
- to use this functionality !</div><br/>
- <pre>import matplotlib.pyplot as plt
- plt.figure()
- (.. ..)
- z.show(plt)
- plt.close()
- </pre>
- <div><br/> z.show function can take optional parameters
- to adapt graph dimensions (width and height) and format (png or svg)</div>
- <div><b>example </b>:
- <pre>z.show(plt,width='50px
- z.show(plt,height='150px', fmt='svg') </pre></div>
-
- <h3>Pandas DataFrame</h3>
- <div> You need to have Pandas module installed
- to use this functionality (pip install pandas) !</div><br/>
- <div>The interpreter can visualize Pandas DataFrame
- with the function z.show()
- <pre>
- import pandas as pd
- df = pd.read_csv("bank.csv", sep=";")
- z.show(df)
- </pre></div>
-
- <h3>SQL over Pandas DataFrame</h3>
- <div> You need to have Pandas&Pandasql modules installed
- to use this functionality (pip install pandas pandasql) !</div><br/>
-
- <div>Python interpreter group includes %sql interpreter that can query
- Pandas DataFrames using SQL and visualize results using Zeppelin Table Display System
-
- <pre>
- %python
- import pandas as pd
- df = pd.read_csv("bank.csv", sep=";")
- </pre>
- <br />
- <pre>
- %python.sql
- %sql
- SELECT * from df LIMIT 5
- </pre>
- </div>
-    """)
-
-
-class PyZeppelinContext(object):
-    """ If py4j is detected, these class will be override
-        with the implementation in bootstrap_input.py
-    """
-    errorMsg = "You must install py4j Python module " \
-               "(pip install py4j) to use Zeppelin dynamic forms features"
-    
-    def __init__(self):
-        self.max_result = 1000
-        self._displayhook = lambda *args: None
-        self._setup_matplotlib()
-    
-    def input(self, name, defaultValue=""):
-        print(self.errorMsg)
-    
-    def select(self, name, options, defaultValue=""):
-        print(self.errorMsg)
-    
-    def checkbox(self, name, options, defaultChecked=[]):
-        print(self.errorMsg)
-    
-    def show(self, p, **kwargs):
-        if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
-            self.show_matplotlib(p, **kwargs)
-        elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
-            # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
-            # and so a dependency on pandas
-            self.show_dataframe(p, **kwargs)
-        elif hasattr(p, '__call__'):
-            p() #error reporting
-    
-    def show_dataframe(self, df, show_index=False, **kwargs):
-        """Pretty prints DF using Table Display System
-        """
-        limit = len(df) > self.max_result
-        header_buf = StringIO("")
-        if show_index:
-            idx_name = str(df.index.name) if df.index.name is not None else ""
-            header_buf.write(idx_name + "\t")
-        header_buf.write(str(df.columns[0]))
-        for col in df.columns[1:]:
-            header_buf.write("\t")
-            header_buf.write(str(col))
-        header_buf.write("\n")
-        
-        body_buf = StringIO("")
-        rows = df.head(self.max_result).values if limit else df.values
-        index = df.index.values
-        for idx, row in zip(index, rows):
-            if show_index:
-                body_buf.write("%html <strong>{}</strong>".format(idx))
-                body_buf.write("\t")
-            body_buf.write(str(row[0]))
-            for cell in row[1:]:
-                body_buf.write("\t")
-                body_buf.write(str(cell))
-            body_buf.write("\n")
-        body_buf.seek(0); header_buf.seek(0)
-        #TODO(bzz): fix it, so it shows red notice, as in Spark
-        print("%table " + header_buf.read() + body_buf.read()) # +
-        #      ("\n<font color=red>Results are limited by {}.</font>" \
-        #          .format(self.max_result) if limit else "")
-        #)
-        body_buf.close(); header_buf.close()
-    
-    def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
-                        **kwargs):
-        """Matplotlib show function
-        """
-        if fmt == "png":
-            img = BytesIO()
-            p.savefig(img, format=fmt)
-            img_str = b"data:image/png;base64,"
-            img_str += base64.b64encode(img.getvalue().strip())
-            img_tag = "<img src={img} style='width={width};height:{height}'>"
-            # Decoding is necessary for Python 3 compability
-            img_str = img_str.decode("ascii")
-            img_str = img_tag.format(img=img_str, width=width, height=height)
-        elif fmt == "svg":
-            img = StringIO()
-            p.savefig(img, format=fmt)
-            img_str = img.getvalue()
-        else:
-            raise ValueError("fmt must be 'png' or 'svg'")
-        
-        html = "%html <div style='width:{width};height:{height}'>{img}<div>"
-        print(html.format(width=width, height=height, img=img_str))
-        img.close()
-    
-    def configure_mpl(self, **kwargs):
-        import mpl_config
-        mpl_config.configure(**kwargs)
-    
-    def _setup_matplotlib(self):
-        # If we don't have matplotlib installed don't bother continuing
-        try:
-            import matplotlib
-        except ImportError:
-            return
-        # Make sure custom backends are available in the PYTHONPATH
-        rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
-        mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
-        if mpl_path not in sys.path:
-            sys.path.append(mpl_path)
-        
-        # Finally check if backend exists, and if so configure as appropriate
-        try:
-            matplotlib.use('module://backend_zinline')
-            import backend_zinline
-            
-            # Everything looks good so make config assuming that we are using
-            # an inline backend
-            self._displayhook = backend_zinline.displayhook
-            self.configure_mpl(width=600, height=400, dpi=72,
-                               fontsize=10, interactive=True, format='png')
-        except ImportError:
-            # Fall back to Agg if no custom backend installed
-            matplotlib.use('Agg')
-            warnings.warn("Unable to load inline matplotlib backend, "
-                          "falling back to Agg")
-
-
-z = PyZeppelinContext()

python/src/main/resources/bootstrap_input.py

@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from py4j.java_gateway import JavaGateway
-from py4j.java_gateway import java_import, JavaGateway, GatewayClient
-
-
-client = GatewayClient(port=%PORT%)
-gateway = JavaGateway(client)
-java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
-
-
-class Py4jZeppelinContext(PyZeppelinContext):
-    """A context impl that uses Py4j to communicate to JVM
-    """
-    def __init__(self, z):
-        PyZeppelinContext.__init__(self)
-        self.z = z
-        self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
-        self.javaList = gateway.jvm.java.util.ArrayList
-        self.max_result = self.z.getMaxResult()
-    
-    def input(self, name, defaultValue=""):
-        return self.z.getGui().input(name, defaultValue)
-    
-    def select(self, name, options, defaultValue=""):
-        javaOptions = gateway.new_array(self.paramOption, len(options))
-        i = 0
-        for tuple in options:
-            javaOptions[i] = self.paramOption(tuple[0], tuple[1])
-            i += 1
-        return self.z.getGui().select(name, defaultValue, javaOptions)
-    
-    def checkbox(self, name, options, defaultChecked=[]):
-        javaOptions = gateway.new_array(self.paramOption, len(options))
-        i = 0
-        for tuple in options:
-            javaOptions[i] = self.paramOption(tuple[0], tuple[1])
-            i += 1
-        javaDefaultCheck = self.javaList()
-        for check in defaultChecked:
-            javaDefaultCheck.append(check)
-        return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
-
-
-z = Py4jZeppelinContext(gateway.entry_point)

python/src/main/resources/interpreter-setting.json

@@ -39,7 +39,7 @@
     "className": "org.apache.zeppelin.python.PythonCondaInterpreter",
     "properties": {
     },
-    "editor":{
+    "editor": {
       "language": "sh",
       "editOnDblClick": false
     }

python/src/main/resources/python/bootstrap_sql.py

@@ -21,8 +21,9 @@
 from __future__ import print_function
 
 try:
-    from pandasql import sqldf
-    pysqldf = lambda q: sqldf(q, globals())
+  from pandasql import sqldf
+  pysqldf = lambda q: sqldf(q, globals())
 except ImportError:
-    pysqldf = lambda q: print("Can not run SQL over Pandas DataFrame" +
-                              "Make sure 'pandas' and 'pandasql' libraries are installed")
+  pysqldf = lambda q: print("Can not run SQL over Pandas DataFrame" +
+                              "Make sure 'pandas' and 'pandasql' libraries are installed")
+

python/src/main/resources/python/zeppelin_python.py

@@ -0,0 +1,293 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, sys, getopt, traceback, json, re
+
+from py4j.java_gateway import java_import, JavaGateway, GatewayClient
+from py4j.protocol import Py4JJavaError, Py4JNetworkError
+import warnings
+import ast
+import traceback
+import warnings
+import signal
+
+from io import BytesIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+# for back compatibility
+
+class Logger(object):
+  def __init__(self):
+    pass
+
+  def write(self, message):
+    intp.appendOutput(message)
+
+  def reset(self):
+    pass
+
+  def flush(self):
+    pass
+
+
+class PyZeppelinContext(object):
+  """ A context impl that uses Py4j to communicate to JVM
+  """
+
+  def __init__(self, z):
+    self.z = z
+    self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
+    self.javaList = gateway.jvm.java.util.ArrayList
+    self.max_result = 1000
+    self._displayhook = lambda *args: None
+    self._setup_matplotlib()
+
+  def getInterpreterContext(self):
+    return self.z.getCurrentInterpreterContext()
+
+  def input(self, name, defaultValue=""):
+    return self.z.getGui().input(name, defaultValue)
+
+  def select(self, name, options, defaultValue=""):
+    javaOptions = gateway.new_array(self.paramOption, len(options))
+    i = 0
+    for tuple in options:
+      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
+      i += 1
+    return self.z.getGui().select(name, defaultValue, javaOptions)
+
+  def checkbox(self, name, options, defaultChecked=[]):
+    javaOptions = gateway.new_array(self.paramOption, len(options))
+    i = 0
+    for tuple in options:
+      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
+      i += 1
+    javaDefaultCheck = self.javaList()
+    for check in defaultChecked:
+      javaDefaultCheck.append(check)
+    return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
+
+  def show(self, p, **kwargs):
+    if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
+      self.show_matplotlib(p, **kwargs)
+    elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
+      # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
+      # and so a dependency on pandas
+      self.show_dataframe(p, **kwargs)
+    elif hasattr(p, '__call__'):
+      p() #error reporting
+
+  def show_dataframe(self, df, show_index=False, **kwargs):
+    """Pretty prints DF using Table Display System
+    """
+    limit = len(df) > self.max_result
+    header_buf = StringIO("")
+    if show_index:
+      idx_name = str(df.index.name) if df.index.name is not None else ""
+      header_buf.write(idx_name + "\t")
+    header_buf.write(str(df.columns[0]))
+    for col in df.columns[1:]:
+      header_buf.write("\t")
+      header_buf.write(str(col))
+    header_buf.write("\n")
+
+    body_buf = StringIO("")
+    rows = df.head(self.max_result).values if limit else df.values
+    index = df.index.values
+    for idx, row in zip(index, rows):
+      if show_index:
+        body_buf.write("%html <strong>{}</strong>".format(idx))
+        body_buf.write("\t")
+      body_buf.write(str(row[0]))
+      for cell in row[1:]:
+        body_buf.write("\t")
+        body_buf.write(str(cell))
+      body_buf.write("\n")
+    body_buf.seek(0); header_buf.seek(0)
+    #TODO(bzz): fix it, so it shows red notice, as in Spark
+    print("%table " + header_buf.read() + body_buf.read()) # +
+    #      ("\n<font color=red>Results are limited by {}.</font>" \
+    #          .format(self.max_result) if limit else "")
+    #)
+    body_buf.close(); header_buf.close()
+
+  def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
+                      **kwargs):
+    """Matplotlib show function
+    """
+    if fmt == "png":
+      img = BytesIO()
+      p.savefig(img, format=fmt)
+      img_str = b"data:image/png;base64,"
+      img_str += base64.b64encode(img.getvalue().strip())
+      img_tag = "<img src={img} style='width={width};height:{height}'>"
+      # Decoding is necessary for Python 3 compability
+      img_str = img_str.decode("ascii")
+      img_str = img_tag.format(img=img_str, width=width, height=height)
+    elif fmt == "svg":
+      img = StringIO()
+      p.savefig(img, format=fmt)
+      img_str = img.getvalue()
+    else:
+      raise ValueError("fmt must be 'png' or 'svg'")
+
+    html = "%html <div style='width:{width};height:{height}'>{img}<div>"
+    print(html.format(width=width, height=height, img=img_str))
+    img.close()
+
+  def configure_mpl(self, **kwargs):
+    import mpl_config
+    mpl_config.configure(**kwargs)
+
+  def _setup_matplotlib(self):
+    # If we don't have matplotlib installed don't bother continuing
+    try:
+      import matplotlib
+    except ImportError:
+      return
+    # Make sure custom backends are available in the PYTHONPATH
+    rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
+    mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
+    if mpl_path not in sys.path:
+      sys.path.append(mpl_path)
+
+    # Finally check if backend exists, and if so configure as appropriate
+    try:
+      matplotlib.use('module://backend_zinline')
+      import backend_zinline
+
+      # Everything looks good so make config assuming that we are using
+      # an inline backend
+      self._displayhook = backend_zinline.displayhook
+      self.configure_mpl(width=600, height=400, dpi=72,
+                         fontsize=10, interactive=True, format='png')
+    except ImportError:
+      # Fall back to Agg if no custom backend installed
+      matplotlib.use('Agg')
+      warnings.warn("Unable to load inline matplotlib backend, "
+                    "falling back to Agg")
+
+
+def handler_stop_signals(sig, frame):
+  sys.exit("Got signal : " + str(sig))
+
+
+signal.signal(signal.SIGINT, handler_stop_signals)
+
+host = "127.0.0.1"
+if len(sys.argv) >= 3:
+  host = sys.argv[2]
+
+client = GatewayClient(address=host, port=int(sys.argv[1]))
+
+#gateway = JavaGateway(client, auto_convert = True)
+gateway = JavaGateway(client)
+
+intp = gateway.entry_point
+intp.onPythonScriptInitialized(os.getpid())
+
+java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
+z = PyZeppelinContext(intp)
+z._setup_matplotlib()
+
+output = Logger()
+sys.stdout = output
+#sys.stderr = output
+
+while True :
+  req = intp.getStatements()
+  if req == None:
+    break
+
+  try:
+    stmts = req.statements().split("\n")
+    final_code = []
+
+    # Get post-execute hooks
+    try:
+      global_hook = intp.getHook('post_exec_dev')
+    except:
+      global_hook = None
+
+    try:
+      user_hook = z.getHook('post_exec')
+    except:
+      user_hook = None
+      
+    nhooks = 0
+    for hook in (global_hook, user_hook):
+      if hook:
+        nhooks += 1
+
+    for s in stmts:
+      if s == None:
+        continue
+
+      # skip comment
+      s_stripped = s.strip()
+      if len(s_stripped) == 0 or s_stripped.startswith("#"):
+        continue
+
+      final_code.append(s)
+
+    if final_code:
+      # use exec mode to compile the statements except the last statement,
+      # so that the last statement's evaluation will be printed to stdout
+      code = compile('\n'.join(final_code), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
+
+      to_run_hooks = []
+      if (nhooks > 0):
+        to_run_hooks = code.body[-nhooks:]
+
+      to_run_exec, to_run_single = (code.body[:-(nhooks + 1)],
+                                    [code.body[-(nhooks + 1)]])
+
+      try:
+        for node in to_run_exec:
+          mod = ast.Module([node])
+          code = compile(mod, '<stdin>', 'exec')
+          exec(code)
+
+        for node in to_run_single:
+          mod = ast.Interactive([node])
+          code = compile(mod, '<stdin>', 'single')
+          exec(code)
+
+        for node in to_run_hooks:
+          mod = ast.Module([node])
+          code = compile(mod, '<stdin>', 'exec')
+          exec(code)
+      except:
+        raise Exception(traceback.format_exc())
+
+    intp.setStatementsFinished("", False)
+  except Py4JJavaError:
+    excInnerError = traceback.format_exc() # format_tb() does not return the inner exception
+    innerErrorStart = excInnerError.find("Py4JJavaError:")
+    if innerErrorStart > -1:
+       excInnerError = excInnerError[innerErrorStart:]
+    intp.setStatementsFinished(excInnerError + str(sys.exc_info()), True)
+  except Py4JNetworkError:
+    # lost connection from gateway server. exit
+    sys.exit(1)
+  except:
+    intp.setStatementsFinished(traceback.format_exc(), True)
+
+  output.reset()

python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java

@@ -1,21 +1,23 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.zeppelin.python;
 
+
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.user.AuthenticationInfo;
@@ -132,4 +134,6 @@ public class PythonCondaInterpreterTest {
         null,
         new InterpreterOutput(null));
   }
+
+
 }

python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java

@@ -21,8 +21,11 @@ import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.net.Inet4Address;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Properties;
@@ -46,8 +49,12 @@ public class PythonDockerInterpreterTest {
     group.put("note", Arrays.asList(python, docker));
     python.setInterpreterGroup(group);
     docker.setInterpreterGroup(group);
+
     doReturn(true).when(docker).pull(any(InterpreterOutput.class), anyString());
     doReturn(python).when(docker).getPythonInterpreter();
+    doReturn("/scriptpath/zeppelin_python.py").when(python).getScriptPath();
+
+    docker.open();
   }
 
   @Test
@@ -57,7 +64,7 @@ public class PythonDockerInterpreterTest {
     verify(python, times(1)).open();
     verify(python, times(1)).close();
     verify(docker, times(1)).pull(any(InterpreterOutput.class), anyString());
-    verify(python).setPythonCommand("docker run -i --rm env python -iu");
+    verify(python).setPythonCommand(Mockito.matches("docker run -i --rm -v.*"));
   }
 
   @Test

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java

@@ -1,25 +1,22 @@
-
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
-import java.util.*;
-
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
@@ -29,34 +26,27 @@ import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.resource.LocalResourcePool;
 import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
 import static org.junit.Assert.*;
 
-/**
- * In order for this test to work, test env must have installed:
- * <ol>
- *  - <li>Python</li>
- *  - <li>Matplotlib</li>
- * <ol>
- *
- * Your PYTHONPATH should also contain the directory of the Matplotlib
- * backend files. Usually these can be found in $ZEPPELIN_HOME/interpreter/lib/python.
- *
- * To run manually on such environment, use:
- * <code>
- *   mvn -Dpython.test.exclude='' test -pl python -am
- * </code>
- */
-public class PythonInterpreterMatplotlibTest {
-
+public class PythonInterpreterMatplotlibTest implements InterpreterOutputListener {
   private InterpreterGroup intpGroup;
   private PythonInterpreter python;
 
   private InterpreterContext context;
+  InterpreterOutput out;
 
   @Before
   public void setUp() throws Exception {
@@ -68,16 +58,27 @@ public class PythonInterpreterMatplotlibTest {
 
     python = new PythonInterpreter(p);
     python.setInterpreterGroup(intpGroup);
-    python.open();
 
     List<Interpreter> interpreters = new LinkedList<>();
     interpreters.add(python);
     intpGroup.put("note", interpreters);
 
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null), null,
-        new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
+    out = new InterpreterOutput(this);
+
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        out);
+    python.open();
+  }
+
+  @After
+  public void afterTest() throws IOException {
+    python.close();
   }
 
   @Test
@@ -85,14 +86,14 @@ public class PythonInterpreterMatplotlibTest {
     // matplotlib
     InterpreterResult ret = python.interpret("import matplotlib", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
-    
+
     // inline backend
     ret = python.interpret("import backend_zinline", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
   }
 
   @Test
-  public void showPlot() {
+  public void showPlot() throws IOException {
     // Simple plot test
     InterpreterResult ret;
     ret = python.interpret("import matplotlib.pyplot as plt", context);
@@ -100,15 +101,16 @@ public class PythonInterpreterMatplotlibTest {
     ret = python.interpret("plt.plot([1, 2, 3])", context);
     ret = python.interpret("plt.show()", context);
 
-    assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(ret.message().get(0).getData(), Type.HTML, ret.message().get(0).getType());
-    assertTrue(ret.message().get(0).getData().contains("data:image/png;base64"));
-    assertTrue(ret.message().get(0).getData().contains("<div>"));
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Type.TEXT, out.getOutputAt(0).getType());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Type.HTML, out.getOutputAt(1).getType());
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("data:image/png;base64"));
+    assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("<div>"));
   }
 
   @Test
   // Test for when configuration is set to auto-close figures after show().
-  public void testClose() {
+  public void testClose() throws IOException {
     InterpreterResult ret;
     InterpreterResult ret1;
     InterpreterResult ret2;
@@ -116,25 +118,33 @@ public class PythonInterpreterMatplotlibTest {
     ret = python.interpret("z.configure_mpl(interactive=False)", context);
     ret = python.interpret("plt.plot([1, 2, 3])", context);
     ret1 = python.interpret("plt.show()", context);
-    
+
     // Second call to show() should print nothing, and Type should be TEXT.
     // This is because when close=True, there should be no living instances
     // of FigureManager, causing show() to return before setting the output
     // type to HTML.
     ret = python.interpret("plt.show()", context);
+
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
     assertEquals(0, ret.message().size());
-    
+
     // Now test that new plot is drawn. It should be identical to the
     // previous one.
     ret = python.interpret("plt.plot([1, 2, 3])", context);
+    String msg1 =  new String(out.getOutputAt(0).toByteArray());
+    InterpreterResult.Type type1 = out.getOutputAt(0).getType();
+
     ret2 = python.interpret("plt.show()", context);
-    assertEquals(ret1.message().get(0).getType(), ret2.message().get(0).getType());
-    assertEquals(ret1.message().get(0).getData(), ret2.message().get(0).getData());
+    String msg2 =  new String(out.getOutputAt(0).toByteArray());
+    InterpreterResult.Type type2 = out.getOutputAt(0).getType();
+
+    assertEquals(msg1, msg2);
+    assertEquals(type1, type2);
   }
-  
+
   @Test
   // Test for when configuration is set to not auto-close figures after show().
-  public void testNoClose() {
+  public void testNoClose() throws IOException {
     InterpreterResult ret;
     InterpreterResult ret1;
     InterpreterResult ret2;
@@ -142,19 +152,39 @@ public class PythonInterpreterMatplotlibTest {
     ret = python.interpret("z.configure_mpl(interactive=False, close=False)", context);
     ret = python.interpret("plt.plot([1, 2, 3])", context);
     ret1 = python.interpret("plt.show()", context);
-    
+
     // Second call to show() should print nothing, and Type should be HTML.
     // This is because when close=False, there should be living instances
     // of FigureManager, causing show() to set the output
     // type to HTML even though the figure is inactive.
     ret = python.interpret("plt.show()", context);
-    assertEquals("", ret.message().get(0).getData());
-    
+    String msg1 =  new String(out.getOutputAt(0).toByteArray());
+    assertNotSame("", msg1);
+
     // Now test that plot can be reshown if it is updated. It should be
     // different from the previous one because it will plot the same line
     // again but in a different color.
     ret = python.interpret("plt.plot([1, 2, 3])", context);
+    msg1 =  new String(out.getOutputAt(1).toByteArray());
     ret2 = python.interpret("plt.show()", context);
-    assertNotSame(ret1.message().get(0).getData(), ret2.message().get(0).getData());
+    String msg2 =  new String(out.getOutputAt(1).toByteArray());
+
+    assertNotSame(msg1, msg2);
+  }
+
+
+  @Override
+  public void onUpdateAll(InterpreterOutput out) {
+
+  }
+
+  @Override
+  public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
+
+  }
+
+  @Override
+  public void onUpdate(int index, InterpreterResultMessageOutput out) {
+
   }
 }

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java

@@ -21,13 +21,16 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.List;
 import java.util.Properties;
 
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
@@ -35,7 +38,10 @@ import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.resource.LocalResourcePool;
 import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -53,13 +59,14 @@ import org.junit.Test;
  *   mvn -Dpython.test.exclude='' test -pl python -am
  * </code>
  */
-public class PythonInterpreterPandasSqlTest {
+public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener {
 
   private InterpreterGroup intpGroup;
   private PythonInterpreterPandasSql sql;
   private PythonInterpreter python;
 
   private InterpreterContext context;
+  InterpreterOutput out;
 
   @Before
   public void setUp() throws Exception {
@@ -78,14 +85,27 @@ public class PythonInterpreterPandasSqlTest {
 
     intpGroup.put("note", Arrays.asList(python, sql));
 
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null), null,
-        new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
+    out = new InterpreterOutput(this);
+
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        out);
+
+    // to make sure python is running.
+    InterpreterResult ret = python.interpret("\n", context);
+    assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
 
-    //important to be last step
     sql.open();
-    //it depends on python interpreter presence in the same group
+  }
+
+  @After
+  public void afterTest() throws IOException {
+    sql.close();
   }
 
   @Test
@@ -97,23 +117,15 @@ public class PythonInterpreterPandasSqlTest {
   @Test
   public void errorMessageIfDependenciesNotInstalled() {
     InterpreterResult ret;
-    // given
-    ret = python.interpret(
-        "pysqldf = lambda q: print('Can not execute SQL as Python dependency is not installed')",
-         context);
-    assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
-
-    // when
     ret = sql.interpret("SELECT * from something", context);
 
-    // then
     assertNotNull(ret);
-    assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
-    assertTrue(ret.message().get(0).getData().contains("dependency is not installed"));
+    assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.ERROR, ret.code());
+    assertTrue(ret.message().get(0).getData().contains("no such table: something"));
   }
 
   @Test
-  public void sqlOverTestDataPrintsTable() {
+  public void sqlOverTestDataPrintsTable() throws IOException {
     InterpreterResult ret;
     // given
     //String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34";
@@ -121,36 +133,34 @@ public class PythonInterpreterPandasSqlTest {
     ret = python.interpret("import numpy as np", context);
     // DataFrame df2 \w test data
     ret = python.interpret("df2 = pd.DataFrame({ 'age'  : np.array([33, 51, 51, 34]), "+
-                           "'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
+        "'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
 
     //when
     ret = sql.interpret("select name, age from df2 where age < 40", context);
 
     //then
-    assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(ret.message().get(0).getData(), Type.TABLE, ret.message().get(0).getType());
-    //assertEquals(expectedTable, ret.message()); //somehow it's same but not equal
-    assertTrue(ret.message().get(0).getData().indexOf("moon\t33") > 0);
-    assertTrue(ret.message().get(0).getData().indexOf("park\t34") > 0);
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("moon\t33") > 0);
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("park\t34") > 0);
 
     assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from df2", context).code());
   }
 
   @Test
-  public void badSqlSyntaxFails() {
+  public void badSqlSyntaxFails() throws IOException {
     //when
     InterpreterResult ret = sql.interpret("select wrong syntax", context);
 
     //then
     assertNotNull("Interpreter returned 'null'", ret);
-    //System.out.println("\nInterpreter response: \n" + ret.message());
     assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
+    assertTrue(out.toInterpreterResultMessage().size() == 0);
   }
 
   @Test
-  public void showDataFrame() {
+  public void showDataFrame() throws IOException {
     InterpreterResult ret;
     ret = python.interpret("import pandas as pd", context);
     ret = python.interpret("import numpy as np", context);
@@ -165,11 +175,25 @@ public class PythonInterpreterPandasSqlTest {
     ret = python.interpret("z.show(df1, show_index=True)", context);
 
     // then
-    assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(ret.message().get(0).getData(), Type.TABLE, ret.message().get(0).getType());
-    assertTrue(ret.message().get(0).getData().indexOf("index_name") == 0);
-    assertTrue(ret.message().get(0).getData().indexOf("13") > 0);
-    assertTrue(ret.message().get(0).getData().indexOf("nan") > 0);
-    assertTrue(ret.message().get(0).getData().indexOf("6.7") > 0);
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("index_name"));
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("nan"));
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("6.7"));
   }
-}
+
+  @Override
+  public void onUpdateAll(InterpreterOutput out) {
+
+  }
+
+  @Override
+  public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
+
+  }
+
+  @Override
+  public void onUpdate(int index, InterpreterResultMessageOutput out) {
+
+  }
+}

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java

@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
@@ -21,52 +21,39 @@ import static org.apache.zeppelin.python.PythonInterpreter.DEFAULT_ZEPPELIN_PYTH
 import static org.apache.zeppelin.python.PythonInterpreter.MAX_RESULT;
 import static org.apache.zeppelin.python.PythonInterpreter.ZEPPELIN_PYTHON;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
+import java.io.File;
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.Map;
 import java.util.Properties;
-import java.util.concurrent.TimeUnit;
 
-import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-
-/**
- * Python interpreter unit test
- *
- * Important: ALL tests here DO NOT REQUIRE Python to be installed
- * If Python dependency is required, please look at PythonInterpreterWithPythonInstalledTest
- */
-public class PythonInterpreterTest {
-  private static final Logger LOG = LoggerFactory.getLogger(PythonProcess.class);
-
-  PythonInterpreter zeppelinPythonInterpreter = null;
+public class PythonInterpreterTest implements InterpreterOutputListener {
   PythonInterpreter pythonInterpreter = null;
-  PythonProcess mockPythonProcess;
   String cmdHistory;
+  private InterpreterContext context;
+  InterpreterOutput out;
 
   public static Properties getPythonTestProperties() {
     Properties p = new Properties();
@@ -79,18 +66,8 @@ public class PythonInterpreterTest {
   public void beforeTest() throws IOException {
     cmdHistory = "";
 
-    /*Mock python process*/
-    mockPythonProcess = mock(PythonProcess.class);
-    when(mockPythonProcess.getPid()).thenReturn(1L);
-    when(mockPythonProcess.sendAndGetResult(anyString())).thenAnswer(new Answer<String>() {
-      @Override public String answer(InvocationOnMock invocationOnMock) throws Throwable {
-        return answerFromPythonMock(invocationOnMock);
-      }
-    });
-
     // python interpreter
-    pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties()));
-    zeppelinPythonInterpreter = new PythonInterpreter(getPythonTestProperties());
+    pythonInterpreter = new PythonInterpreter(getPythonTestProperties());
 
     // create interpreter group
     InterpreterGroup group = new InterpreterGroup();
@@ -98,186 +75,49 @@ public class PythonInterpreterTest {
     group.get("note").add(pythonInterpreter);
     pythonInterpreter.setInterpreterGroup(group);
 
-    when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess);
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError");
+    out = new InterpreterOutput(this);
+
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(group.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        out);
+    pythonInterpreter.open();
   }
 
   @After
   public void afterTest() throws IOException {
     pythonInterpreter.close();
-    zeppelinPythonInterpreter.close();
   }
 
   @Test
-  public void testOpenInterpreter() {
-    pythonInterpreter.open();
-    assertEquals(pythonInterpreter.getPythonProcess().getPid(), 1);
-  }
-
-  /**
-   * If Py4J is not installed, bootstrap_input.py
-   * is not sent to Python process and py4j JavaGateway is not running
-   */
-  @Test
-  public void testPy4jIsNotInstalled() {
-    pythonInterpreter.open();
-    assertNull(pythonInterpreter.getPy4jPort());
-    assertTrue(cmdHistory.contains("def help()"));
-    assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
-    assertTrue(cmdHistory.contains("z = PyZeppelinContext"));
-    assertTrue(cmdHistory.contains("def show"));
-    assertFalse(cmdHistory.contains("GatewayClient"));
-  }
-
-  /**
-   * If Py4J installed, bootstrap_input.py
-   * is sent to interpreter and JavaGateway is running
-   */
-  @Test
-  public void testPy4jInstalled() throws IOException, InterruptedException {
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
-
-    pythonInterpreter.open();
-    Integer py4jPort = pythonInterpreter.getPy4jPort();
-    assertNotNull(py4jPort);
-
-    assertTrue(cmdHistory.contains("def help()"));
-    assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
-    assertTrue(cmdHistory.contains("z = Py4jZeppelinContext"));
-    assertTrue(cmdHistory.contains("def show"));
-    assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")"));
-    assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input"));
-
-    assertTrue(serverIsListeningOn(py4jPort));
-    pythonInterpreter.close();
-    TimeUnit.MILLISECONDS.sleep(100);
-    assertFalse(serverIsListeningOn(py4jPort));
-  }
-
-  @Test
-  public void testClose() throws IOException, InterruptedException {
-    //given: py4j is installed
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
-
-    pythonInterpreter.open();
-    Integer py4jPort = pythonInterpreter.getPy4jPort();
-    assertNotNull(py4jPort);
-
-    //when
-    pythonInterpreter.close();
-    TimeUnit.MILLISECONDS.sleep(100);
-
-    //then
-    assertFalse(serverIsListeningOn(py4jPort));
-    verify(mockPythonProcess, times(1)).close();
-  }
-
-  @Test
-  public void testInterpret() {
-    pythonInterpreter.open();
-    cmdHistory = "";
-    InterpreterResult result = pythonInterpreter.interpret("print a", null);
+  public void testInterpret() throws InterruptedException, IOException {
+    InterpreterResult result = pythonInterpreter.interpret("print (\"hi\")", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    assertEquals("%text print a", result.message().get(0).toString());
   }
 
   @Test
-  public void testInterpretInvalidSyntax() {
-    zeppelinPythonInterpreter.open();
-    InterpreterResult result = zeppelinPythonInterpreter.interpret("for x in range(0,3):  print (\"hi\")\n\nz._displayhook()", null);
+  public void testInterpretInvalidSyntax() throws IOException {
+    InterpreterResult result = pythonInterpreter.interpret("for x in range(0,3):  print (\"hi\")\n", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    assertTrue(result.message().get(0).toString().contains("hi\nhi\nhi"));
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
+ }
+
+  @Override
+  public void onUpdateAll(InterpreterOutput out) {
 
-    result = zeppelinPythonInterpreter.interpret("for x in range(0,3):  print (\"hi\")\nz._displayhook()", null);
-    assertEquals(InterpreterResult.Code.ERROR, result.code());
-    assertTrue(result.message().get(0).toString().contains("SyntaxError: invalid syntax"));
   }
 
-  /**
-   * Checks if given port is open on 'localhost'
-   * @param port
-   */
-  private boolean serverIsListeningOn(Integer port) {
-    Socket s = new Socket();
-    boolean serverIsListening = false;
+  @Override
+  public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
 
-    int retryCount = 0;
-    boolean connected = false;
-    while (connected = tryToConnect(s, port) && retryCount < 10) {
-      serverIsListening = connected;
-      tryToClose(s);
-      retryCount++;
-      s = new Socket();
-    }
-    return serverIsListening;
   }
 
-  private boolean tryToConnect(Socket s, Integer port) {
-    boolean connected = false;
-    SocketAddress sa = new InetSocketAddress("localhost", port);
-    try {
-      s.connect(sa, 10000);
-      connected = true;
-    } catch (IOException e) {
-      //LOG.warn("Can't open connection to " + sa, e);
-    }
-    return connected;
+  @Override
+  public void onUpdate(int index, InterpreterResultMessageOutput out) {
+
   }
-
-  private void tryToClose(Socket s) {
-    try {
-      s.close();
-    } catch (IOException e) {
-      LOG.error("Can't close connection to " + s.getInetAddress(), e);
-    }
-  }
-
-  private String answerFromPythonMock(InvocationOnMock invocationOnMock) {
-    Object[] inputs = invocationOnMock.getArguments();
-    String cmdToExecute = (String) inputs[0];
-
-    if (cmdToExecute != null) {
-      cmdHistory += cmdToExecute;
-      String[] lines = cmdToExecute.split("\\n");
-      String output = "";
-
-      for (int i = 0; i < lines.length; i++) {
-        output += lines[i];
-      }
-      return output;
-    } else {
-      return "";
-    }
-  }
-
-  @Test
-  public void checkMultiRowErrorFails() {
-
-    PythonInterpreter pythonInterpreter = new PythonInterpreter(
-      PythonInterpreterTest.getPythonTestProperties()
-    );
-    // create interpreter group
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", new LinkedList<Interpreter>());
-    group.get("note").add(pythonInterpreter);
-    pythonInterpreter.setInterpreterGroup(group);
-
-    pythonInterpreter.open();
-
-    String codeRaiseException = "raise Exception(\"test exception\")";
-    InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
-
-    assertNotNull("Interpreter result for raise exception is Null", ret);
-
-    System.err.println("ret = '" + ret + "'");
-    assertEquals(InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    assertNotNull("Interpreter result for text is Null", ret);
-    String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
-    ret = pythonInterpreter.interpret(codePrintText, null);
-    assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-  }
-
 }

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java

@@ -1,125 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.zeppelin.python;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-/**
- * Python interpreter unit test that user real Python
- *
- * Important: ALL tests here REQUIRE Python to be installed
- * They are excluded from default build, to run them manually do:
- *
- * <code>
- * mvn "-Dtest=org.apache.zeppelin.python.PythonInterpreterWithPythonInstalledTest" test -pl python
- * </code>
- *
- * or
- * <code>
- * mvn -Dpython.test.exclude='' test -pl python -am
- * </code>
- */
-public class PythonInterpreterWithPythonInstalledTest {
-
-  @Test
-  public void badPythonSyntaxFails() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-        PythonInterpreterTest.getPythonTestProperties());
-    // create interpreter group
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-
-    realPython.open();
-
-    //when
-    InterpreterResult ret = realPython.interpret("select wrong syntax", null);
-
-    //then
-    assertNotNull("Interpreter returned 'null'", ret);
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    realPython.close();
-  }
-
-  @Test
-  public void goodPythonSyntaxRuns() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-        PythonInterpreterTest.getPythonTestProperties());
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-    realPython.open();
-
-    //when
-    InterpreterResult ret = realPython.interpret("help()", null);
-
-    //then
-    assertNotNull("Interpreter returned 'null'", ret);
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    realPython.close();
-  }
-
-  @Test
-  public void testZeppelin1555() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-            PythonInterpreterTest.getPythonTestProperties());
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-    realPython.open();
-
-    //when
-    InterpreterResult ret1 = realPython.interpret("print(\"...\")", null);
-
-    //then
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.SUCCESS, ret1.code());
-    assertEquals("...\n", ret1.message().get(0).getData());
-
-
-    InterpreterResult ret2 = realPython.interpret("for i in range(5):", null);
-    //then
-    //System.out.println("\nInterpreterResultterpreter response: \n" + ret2.message());
-    assertEquals(InterpreterResult.Code.ERROR, ret2.code());
-    assertEquals("   File \"<stdin>\", line 2\n" +
-            "    \n" +
-            "    ^\n" +
-            "IndentationError: expected an indented block\n", ret2.message().get(0).getData());
-
-    realPython.close();
-  }
-
-}

spark/src/main/resources/python/zeppelin_pyspark.py

@@ -83,6 +83,9 @@ class PyZeppelinContext(dict):
   def get(self, key):
     return self.__getitem__(key)
 
+  def getInterpreterContext(self):
+    return self.z.getInterpreterContext()
+
   def input(self, name, defaultValue=""):
     return self.z.input(name, defaultValue)
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java

@@ -67,7 +67,15 @@ public class Booter {
   }
 
   public static RemoteRepository newCentralRepository() {
-    return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/");
+    String mvnRepo = System.getenv("ZEPPELIN_INTERPRETER_DEP_MVNREPO");
+    if (mvnRepo == null) {
+      mvnRepo = System.getProperty("zeppelin.interpreter.dep.mvnRepo");
+    }
+    if (mvnRepo == null) {
+      mvnRepo = "http://repo1.maven.org/maven2/";
+    }
+
+    return new RemoteRepository("central", "default", mvnRepo);
   }
 
   public static RemoteRepository newLocalRepository() {

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -18,15 +18,19 @@
 package org.apache.zeppelin.interpreter;
 
 
+import java.lang.reflect.Field;
 import java.net.URL;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-import org.apache.zeppelin.annotation.ZeppelinApi;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.zeppelin.annotation.Experimental;
+import org.apache.zeppelin.annotation.ZeppelinApi;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -157,6 +161,8 @@ public abstract class Interpreter {
       }
     }
 
+    replaceContextParameters(p);
+
     return p;
   }
 
@@ -296,6 +302,41 @@ public abstract class Interpreter {
     return null;
   }
 
+  /**
+   * Replace markers #{contextFieldName} by values from {@link InterpreterContext} fields
+   * with same name and marker #{user}. If value == null then replace by empty string.
+   */
+  private void replaceContextParameters(Properties properties) {
+    InterpreterContext interpreterContext = InterpreterContext.get();
+    if (interpreterContext != null) {
+      String markerTemplate = "#\\{%s\\}";
+      List<String> skipFields = Arrays.asList("paragraphTitle", "paragraphId", "paragraphText");
+      List typesToProcess = Arrays.asList(String.class, Double.class, Float.class, Short.class,
+          Byte.class, Character.class, Boolean.class, Integer.class, Long.class);
+      for (String key : properties.stringPropertyNames()) {
+        String p = properties.getProperty(key);
+        if (StringUtils.isNotEmpty(p)) {
+          for (Field field : InterpreterContext.class.getDeclaredFields()) {
+            Class clazz = field.getType();
+            if (!skipFields.contains(field.getName()) && (typesToProcess.contains(clazz)
+                || clazz.isPrimitive())) {
+              Object value = null;
+              try {
+                value = FieldUtils.readField(field, interpreterContext, true);
+              } catch (Exception e) {
+                logger.error("Cannot read value of field {0}", field.getName());
+              }
+              p = p.replaceAll(String.format(markerTemplate, field.getName()),
+                  value != null ? value.toString() : StringUtils.EMPTY);
+            }
+          }
+          p = p.replaceAll(String.format(markerTemplate, "user"),
+              StringUtils.defaultString(userName, StringUtils.EMPTY));
+          properties.setProperty(key, p);
+        }
+      }
+    }
+  }
 
   /**
    * Type of interpreter.

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java

@@ -192,4 +192,8 @@ public class InterpreterContext {
   public void setRemoteWorksController(RemoteWorksController remoteWorksController) {
     this.remoteWorksController = remoteWorksController;
   }
+
+  public InterpreterOutput out() {
+    return out;
+  }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java

@@ -17,15 +17,21 @@
 
 package org.apache.zeppelin.interpreter;
 
-import java.util.*;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.log4j.Logger;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
 import org.apache.zeppelin.resource.ResourcePool;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * InterpreterGroup is list of interpreters in the same interpreter group.
@@ -43,7 +49,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
 public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
   String id;
 
-  Logger LOGGER = Logger.getLogger(InterpreterGroup.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
 
   AngularObjectRegistry angularObjectRegistry;
   InterpreterHookRegistry hookRegistry;
@@ -165,47 +171,71 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
    */
   public void close(String sessionId) {
     LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
-    List<Interpreter> intpForSession = this.get(sessionId);
-    close(intpForSession);
+    final List<Interpreter> intpForSession = this.get(sessionId);
 
-    if (remoteInterpreterProcess != null) {
-      remoteInterpreterProcess.dereference();
-      if (remoteInterpreterProcess.referenceCount() <= 0) {
-        remoteInterpreterProcess = null;
-        allInterpreterGroups.remove(id);
-      }
-    }
+    close(intpForSession);
   }
 
-  private void close(Collection<Interpreter> intpToClose) {
+  private void close(final Collection<Interpreter> intpToClose) {
+    close(null, null, null, intpToClose);
+  }
+
+  public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
+      final String processKey, final String sessionKey) {
+    LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
+    close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
+  }
+
+  private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
+      final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
     if (intpToClose == null) {
       return;
     }
-    List<Thread> closeThreads = new LinkedList<>();
+    Thread t = new Thread() {
+      public void run() {
+        for (Interpreter interpreter : intpToClose) {
+          Scheduler scheduler = interpreter.getScheduler();
+          interpreter.close();
 
-    for (final Interpreter intp : intpToClose) {
-      Thread t = new Thread() {
-        public void run() {
-          Scheduler scheduler = intp.getScheduler();
-          intp.close();
-
-          if (scheduler != null) {
+          if (null != scheduler) {
             SchedulerFactory.singleton().removeScheduler(scheduler.getName());
           }
         }
-      };
 
-      t.start();
-      closeThreads.add(t);
-    }
+        if (remoteInterpreterProcess != null) {
+          //TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
+          // refernceCount is a proper value. And as the same reason, we must not call
+          // remoteInterpreterProcess.dereference twice - this method also be called by
+          // interpreter.close().
 
-    for (Thread t : closeThreads) {
-      try {
-        t.join();
-      } catch (InterruptedException e) {
-        LOGGER.error("Can't close interpreter", e);
+          // remoteInterpreterProcess.dereference();
+          if (remoteInterpreterProcess.referenceCount() <= 0) {
+            remoteInterpreterProcess = null;
+            allInterpreterGroups.remove(id);
+          }
+        }
+
+        // TODO(jl): While closing interpreters in a same session, we should remove after all
+        // interpreters are removed. OMG. It's too dirty!!
+        if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
+          InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
+          if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
+            interpreterGroupRef.remove(processKey);
+          } else {
+            interpreterGroup.remove(sessionKey);
+          }
+        }
       }
+    };
+
+    t.start();
+    try {
+      t.join();
+    } catch (InterruptedException e) {
+      LOGGER.error("Can't close interpreter: {}", getId(), e);
     }
+
+
   }
 
   /**

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java

@@ -21,6 +21,7 @@ import java.net.URL;
 import java.util.List;
 import java.util.Properties;
 
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Scheduler;
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java

@@ -26,7 +26,6 @@ import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.helium.ApplicationEventListener;
 import org.apache.zeppelin.display.Input;
 import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
@@ -64,6 +63,7 @@ public class RemoteInterpreter extends Interpreter {
   private String userName;
   private Boolean isUserImpersonate;
   private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
+  private String interpreterGroupName;
 
   /**
    * Remote interpreter and manage interpreter process
@@ -72,7 +72,7 @@ public class RemoteInterpreter extends Interpreter {
       String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
       int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
       ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
-      int outputLimit) {
+      int outputLimit, String interpreterGroupName) {
     super(property);
     this.sessionKey = sessionKey;
     this.className = className;
@@ -88,6 +88,7 @@ public class RemoteInterpreter extends Interpreter {
     this.userName = userName;
     this.isUserImpersonate = isUserImpersonate;
     this.outputLimit = outputLimit;
+    this.interpreterGroupName = interpreterGroupName;
   }
 
 
@@ -185,7 +186,7 @@ public class RemoteInterpreter extends Interpreter {
           // create new remote process
           remoteProcess = new RemoteInterpreterManagedProcess(
               interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
-              remoteInterpreterProcessListener, applicationEventListener);
+              remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
         }
 
         intpGroup.setRemoteInterpreterProcess(remoteProcess);
@@ -203,7 +204,7 @@ public class RemoteInterpreter extends Interpreter {
     RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
 
     final InterpreterGroup interpreterGroup = getInterpreterGroup();
-    interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
+
     interpreterProcess.setMaxPoolSize(
         Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
     String groupId = interpreterGroup.getId();
@@ -251,6 +252,19 @@ public class RemoteInterpreter extends Interpreter {
     synchronized (interpreterGroup) {
       // initialize all interpreters in this interpreter group
       List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+      // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
+      // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
+      // doesn't call open method, it's not open. It causes problem while running intp.close()
+      // In case of Spark, this method initializes all of interpreters and init() method increases
+      // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
+      // other interpreters doesn't do anything because those LazyInterpreters aren't open.
+      // But for now, we have to initialise all of interpreters for some reasons.
+      // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
+      RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
+      if (!initialized) {
+        // reference per session
+        interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
+      }
       for (Interpreter intp : new ArrayList<>(interpreters)) {
         Interpreter p = intp;
         while (p instanceof WrappedInterpreter) {
@@ -269,8 +283,43 @@ public class RemoteInterpreter extends Interpreter {
 
   @Override
   public void close() {
-    RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
+    InterpreterGroup interpreterGroup = getInterpreterGroup();
+    synchronized (interpreterGroup) {
+      // close all interpreters in this session
+      List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
+      // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
+      // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
+      // doesn't call open method, it's not open. It causes problem while running intp.close()
+      // In case of Spark, this method initializes all of interpreters and init() method increases
+      // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
+      // other interpreters doesn't do anything because those LazyInterpreters aren't open.
+      // But for now, we have to initialise all of interpreters for some reasons.
+      // See Interpreter.getInterpreterInTheSameSessionByClassName(String)
+      if (initialized) {
+        // dereference per session
+        getInterpreterProcess().dereference();
+      }
+      for (Interpreter intp : new ArrayList<>(interpreters)) {
+        Interpreter p = intp;
+        while (p instanceof WrappedInterpreter) {
+          p = ((WrappedInterpreter) p).getInnerInterpreter();
+        }
+        try {
+          ((RemoteInterpreter) p).closeInterpreter();
+        } catch (InterpreterException e) {
+          logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
+              p.getClassName());
+          interpreters.remove(p);
+        }
+      }
+    }
+  }
 
+  public void closeInterpreter() {
+    if (this.initialized == false) {
+      return;
+    }
+    RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
     Client client = null;
     boolean broken = false;
     try {
@@ -287,7 +336,7 @@ public class RemoteInterpreter extends Interpreter {
       if (client != null) {
         interpreterProcess.releaseClient(client, broken);
       }
-      getInterpreterProcess().dereference();
+      this.initialized = false;
     }
   }
 
@@ -379,7 +428,7 @@ public class RemoteInterpreter extends Interpreter {
 
   @Override
   public FormType getFormType() {
-    init();
+    open();
 
     if (formType != null) {
       return formType;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java

@@ -44,6 +44,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
   private int port = -1;
   private final String interpreterDir;
   private final String localRepoDir;
+  private final String interpreterGroupName;
 
   private Map<String, String> env;
 
@@ -54,14 +55,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
       Map<String, String> env,
       int connectTimeout,
       RemoteInterpreterProcessListener listener,
-      ApplicationEventListener appListener) {
+      ApplicationEventListener appListener,
+      String interpreterGroupName) {
     super(new RemoteInterpreterEventPoller(listener, appListener),
         connectTimeout);
     this.interpreterRunner = intpRunner;
     this.env = env;
     this.interpreterDir = intpDir;
     this.localRepoDir = localRepoDir;
-
+    this.interpreterGroupName = interpreterGroupName;
   }
 
   RemoteInterpreterManagedProcess(String intpRunner,
@@ -69,13 +71,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
                                   String localRepoDir,
                                   Map<String, String> env,
                                   RemoteInterpreterEventPoller remoteInterpreterEventPoller,
-                                  int connectTimeout) {
+                                  int connectTimeout,
+                                  String interpreterGroupName) {
     super(remoteInterpreterEventPoller,
         connectTimeout);
     this.interpreterRunner = intpRunner;
     this.env = env;
     this.interpreterDir = intpDir;
     this.localRepoDir = localRepoDir;
+    this.interpreterGroupName = interpreterGroupName;
   }
 
   @Override
@@ -108,6 +112,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
     }
     cmdLine.addArgument("-l", false);
     cmdLine.addArgument(localRepoDir, false);
+    cmdLine.addArgument("-g", false);
+    cmdLine.addArgument(interpreterGroupName, false);
 
     executor = new DefaultExecutor();
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java

@@ -31,6 +31,8 @@ import java.util.concurrent.atomic.AtomicInteger;
  */
 public abstract class RemoteInterpreterProcess {
   private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
+
+  // number of sessions that are attached to this process
   private final AtomicInteger referenceCount;
 
   private GenericObjectPool<Client> clientPool;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -276,17 +276,18 @@ public class RemoteInterpreterServer
     }
 
     // close interpreters
+    List<Interpreter> interpreters;
     synchronized (interpreterGroup) {
-      List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
-      if (interpreters != null) {
-        Iterator<Interpreter> it = interpreters.iterator();
-        while (it.hasNext()) {
-          Interpreter inp = it.next();
-          if (inp.getClassName().equals(className)) {
-            inp.close();
-            it.remove();
-            break;
-          }
+      interpreters = interpreterGroup.get(sessionKey);
+    }
+    if (interpreters != null) {
+      Iterator<Interpreter> it = interpreters.iterator();
+      while (it.hasNext()) {
+        Interpreter inp = it.next();
+        if (inp.getClassName().equals(className)) {
+          inp.close();
+          it.remove();
+          break;
         }
       }
     }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java

@@ -76,7 +76,7 @@ public abstract class Job {
 
   transient boolean aborted = false;
 
-  String errorMessage;
+  private String errorMessage;
   private transient Throwable exception;
   private transient JobListener listener;
   private long progressUpdateIntervalMs;
@@ -174,20 +174,14 @@ public abstract class Job {
 
   public void run() {
     JobProgressPoller progressUpdator = null;
+    dateStarted = new Date();
     try {
       progressUpdator = new JobProgressPoller(this, progressUpdateIntervalMs);
       progressUpdator.start();
-      dateStarted = new Date();
-      setResult(jobRun());
-      this.exception = null;
-      errorMessage = null;
-      dateFinished = new Date();
+      completeWithSuccess(jobRun());
     } catch (Throwable e) {
       LOGGER.error("Job failed", e);
-      this.exception = e;
-      setResult(e.getMessage());
-      errorMessage = getStack(e);
-      dateFinished = new Date();
+      completeWithError(e);
     } finally {
       if (progressUpdator != null) {
         progressUpdator.interrupt();
@@ -196,6 +190,19 @@ public abstract class Job {
     }
   }
 
+  private synchronized void completeWithSuccess(Object result) {
+    setResult(result);
+    exception = null;
+    errorMessage = null;
+    dateFinished = new Date();
+  }
+
+  private synchronized void completeWithError(Throwable error) {
+    setResult(error.getMessage());
+    setException(error);
+    dateFinished = new Date();
+  }
+
   public static String getStack(Throwable e) {
     if (e == null) {
       return "";
@@ -209,11 +216,11 @@ public abstract class Job {
     }
   }
 
-  public Throwable getException() {
+  public synchronized Throwable getException() {
     return exception;
   }
 
-  protected void setException(Throwable t) {
+  protected synchronized void setException(Throwable t) {
     exception = t;
     errorMessage = getStack(t);
   }
@@ -252,13 +259,17 @@ public abstract class Job {
     return dateStarted;
   }
 
-  public Date getDateFinished() {
+  public synchronized Date getDateFinished() {
     return dateFinished;
   }
 
   public abstract void setResult(Object results);
 
-  public void setErrorMessage(String errorMessage) {
+  public synchronized String getErrorMessage() {
+    return errorMessage;
+  }
+
+  public synchronized void setErrorMessage(String errorMessage) {
     this.errorMessage = errorMessage;
   }
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java

@@ -17,13 +17,14 @@
 
 package org.apache.zeppelin.interpreter;
 
-import static org.junit.Assert.assertEquals;
-
 import java.util.Properties;
 
 import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
+import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+
 public class InterpreterTest {
 
   @Test
@@ -51,4 +52,38 @@ public class InterpreterTest {
     assertEquals("v2", intp.getProperty("p1"));
   }
 
+  @Test
+  public void testPropertyWithReplacedContextFields() {
+    String noteId = "testNoteId";
+    String paragraphTitle = "testParagraphTitle";
+    String paragraphText = "testParagraphText";
+    String paragraphId = "testParagraphId";
+    String user = "username";
+    InterpreterContext.set(new InterpreterContext(noteId,
+        paragraphId,
+        null,
+        paragraphTitle,
+        paragraphText,
+        new AuthenticationInfo("testUser", "testTicket"),
+        null,
+        null,
+        null,
+        null,
+        null,
+        null));
+    Properties p = new Properties();
+    p.put("p1", "replName #{noteId}, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, #{replName}, #{noteId}, #{user}," +
+        " #{authenticationInfo}");
+    MockInterpreterA intp = new MockInterpreterA(p);
+    intp.setUserName(user);
+    String actual = intp.getProperty("p1");
+    InterpreterContext.remove();
+
+    assertEquals(
+        String.format("replName %s, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, , %s, %s, #{authenticationInfo}", noteId,
+            noteId, user),
+        actual
+    );
+  }
+
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java

@@ -44,7 +44,7 @@ public class RemoteInterpreterProcessTest {
     InterpreterGroup intpGroup = new InterpreterGroup();
     RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
         INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
-        10 * 1000, null, null);
+        10 * 1000, null, null,"fakeName");
     assertFalse(rip.isRunning());
     assertEquals(0, rip.referenceCount());
     assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@@ -61,7 +61,7 @@ public class RemoteInterpreterProcessTest {
     InterpreterGroup intpGroup = new InterpreterGroup();
     RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
         INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
-        mock(RemoteInterpreterEventPoller.class), 10 * 1000);
+        mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
     rip.reference(intpGroup, "anonymous", false);
     assertEquals(0, rip.getNumActiveClient());
     assertEquals(0, rip.getNumIdleClient());
@@ -104,7 +104,8 @@ public class RemoteInterpreterProcessTest {
         "fakeRepo",
         new HashMap<String, String>(),
         mock(RemoteInterpreterEventPoller.class)
-        , 10 * 1000);
+        , 10 * 1000,
+        "fakeName");
     assertFalse(rip.isRunning());
     assertEquals(0, rip.referenceCount());
     assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@@ -117,7 +118,7 @@ public class RemoteInterpreterProcessTest {
     InterpreterGroup intpGroup = new InterpreterGroup();
     RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
         "echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
-        10 * 1000, null, null);
+        10 * 1000, null, null, "fakeName");
     assertFalse(rip.isRunning());
     assertEquals(0, rip.referenceCount());
     try {

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java

@@ -142,7 +142,7 @@ public class RemoteInterpreterTest {
     intpA.open(); // initializa all interpreters in the same group
     assertTrue(process.isRunning());
     assertEquals(1, process.getNumIdleClient());
-    assertEquals(2, process.referenceCount());
+    assertEquals(1, process.referenceCount());
 
     intpA.interpret("1",
         new InterpreterContext(
@@ -159,10 +159,10 @@ public class RemoteInterpreterTest {
             new LinkedList<InterpreterContextRunner>(), null));
 
     intpB.open();
-    assertEquals(2, process.referenceCount());
+    assertEquals(1, process.referenceCount());
 
     intpA.close();
-    assertEquals(1, process.referenceCount());
+    assertEquals(0, process.referenceCount());
     intpB.close();
     assertEquals(0, process.referenceCount());
 

zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java

@@ -38,6 +38,7 @@ import org.apache.shiro.authz.AuthorizationInfo;
 import org.apache.shiro.realm.AuthorizingRealm;
 import org.apache.shiro.subject.PrincipalCollection;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserSessionContainer;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
 import org.apache.zeppelin.server.ZeppelinServer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -155,15 +156,8 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
       throw new AuthenticationException("Cannot login to ZeppelinHub");
     }
 
-    // Add ZeppelinHub user_session token this singleton map, this will help ZeppelinHubRepo
-    // to get specific information about the current user.
-    UserSessionContainer.instance.setSession(account.login, userSession);
-
-    /* TODO(khalid): add proper roles and add listener */
-    HashSet<String> userAndRoles = new HashSet<String>();
-    userAndRoles.add(account.login);
-    ZeppelinServer.notebookWsServer.broadcastReloadedNoteList(
-        new org.apache.zeppelin.user.AuthenticationInfo(account.login), userAndRoles);
+    onLoginSuccess(account.login, userSession);
+    
     return account;
   }
 
@@ -213,4 +207,21 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
     public String email;
     public String name;
   }
+  
+  public void onLoginSuccess(String username, String session) {
+    UserSessionContainer.instance.setSession(username, session);
+
+    /* TODO(xxx): add proper roles */
+    HashSet<String> userAndRoles = new HashSet<String>();
+    userAndRoles.add(username);
+    ZeppelinServer.notebookWsServer.broadcastReloadedNoteList(
+        new org.apache.zeppelin.user.AuthenticationInfo(username), userAndRoles);
+
+    ZeppelinhubUtils.userLoginRoutine(username);
+  }
+  
+  @Override
+  public void onLogout(PrincipalCollection principals) {
+    ZeppelinhubUtils.userLogoutRoutine((String) principals.getPrimaryPrincipal());
+  }
 }

zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java

@@ -190,7 +190,11 @@ public class InterpreterRestApi {
       RestartInterpreterRequest request = gson.fromJson(message, RestartInterpreterRequest.class);
 
       String noteId = request == null ? null : request.getNoteId();
-      interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
+      if (null == noteId) {
+        interpreterSettingManager.close(setting);
+      } else {
+        interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
+      }
       notebookServer.clearParagraphRuntimeInfo(setting);
 
     } catch (InterpreterException e) {

zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java

@@ -18,11 +18,7 @@
 package org.apache.zeppelin.rest;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
@@ -337,16 +333,16 @@ public class NotebookRestApi {
   @Path("/")
   @ZeppelinApi
   public Response createNote(String message) throws IOException {
+    String user = SecurityUtils.getPrincipal();
     LOG.info("Create new note by JSON {}", message);
     NewNoteRequest request = gson.fromJson(message, NewNoteRequest.class);
-    AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+    AuthenticationInfo subject = new AuthenticationInfo(user);
     Note note = notebook.createNote(subject);
     List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
     if (initialParagraphs != null) {
       for (NewParagraphRequest paragraphRequest : initialParagraphs) {
         Paragraph p = note.addParagraph(subject);
-        p.setTitle(paragraphRequest.getTitle());
-        p.setText(paragraphRequest.getText());
+        initParagraph(p, paragraphRequest, user);
       }
     }
     note.addParagraph(subject); // add one paragraph to the last
@@ -425,6 +421,7 @@ public class NotebookRestApi {
   @ZeppelinApi
   public Response insertParagraph(@PathParam("noteId") String noteId, String message)
       throws IOException {
+    String user = SecurityUtils.getPrincipal();
     LOG.info("insert paragraph {} {}", noteId, message);
 
     Note note = notebook.getNote(noteId);
@@ -432,7 +429,7 @@ public class NotebookRestApi {
     checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note");
 
     NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
-    AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
+    AuthenticationInfo subject = new AuthenticationInfo(user);
     Paragraph p;
     Double indexDouble = request.getIndex();
     if (indexDouble == null) {
@@ -440,9 +437,7 @@ public class NotebookRestApi {
     } else {
       p = note.insertParagraph(indexDouble.intValue(), subject);
     }
-    p.setTitle(request.getTitle());
-    p.setText(request.getText());
-
+    initParagraph(p, request, user);
     note.persist(subject);
     notebookServer.broadcastNote(note);
     return new JsonResponse<>(Status.OK, "", p.getId()).build();
@@ -486,17 +481,7 @@ public class NotebookRestApi {
     checkIfParagraphIsNotNull(p);
 
     Map<String, Object> newConfig = gson.fromJson(message, HashMap.class);
-    if (newConfig == null || newConfig.isEmpty()) {
-      LOG.warn("{} is trying to update paragraph {} of note {} with empty config",
-          user, paragraphId, noteId);
-      throw new BadRequestException("paragraph config cannot be empty");
-    }
-    Map<String, Object> origConfig = p.getConfig();
-    for (String key : newConfig.keySet()) {
-      origConfig.put(key, newConfig.get(key));
-    }
-
-    p.setConfig(origConfig);
+    configureParagraph(p, newConfig, user);
     AuthenticationInfo subject = new AuthenticationInfo(user);
     note.persist(subject);
 
@@ -963,4 +948,31 @@ public class NotebookRestApi {
     }
   }
 
+  private void initParagraph(Paragraph p, NewParagraphRequest request, String user)
+      throws IOException {
+    LOG.info("Init Paragraph for user {}", user);
+    checkIfParagraphIsNotNull(p);
+    p.setTitle(request.getTitle());
+    p.setText(request.getText());
+    Map< String, Object > config = request.getConfig();
+    if ( config != null && !config.isEmpty()) {
+      configureParagraph(p, config, user);
+    }
+  }
+
+  private void configureParagraph(Paragraph p, Map< String, Object> newConfig, String user)
+      throws IOException {
+    LOG.info("Configure Paragraph for user {}", user);
+    if (newConfig == null || newConfig.isEmpty()) {
+      LOG.warn("{} is trying to update paragraph {} of note {} with empty config",
+              user, p.getId(), p.getNote().getId());
+      throw new BadRequestException("paragraph config cannot be empty");
+    }
+    Map<String, Object> origConfig = p.getConfig();
+    for (String key : newConfig.keySet()) {
+      origConfig.put(key, newConfig.get(key));
+    }
+    p.setConfig(origConfig);
+  }
+
 }

zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewParagraphRequest.java

@@ -17,15 +17,21 @@
 
 package org.apache.zeppelin.rest.message;
 
+import java.util.HashMap;
+
 /**
  * NewParagraphRequest rest api request message
  *
  * index field will be ignored when it's used to provide initial paragraphs
+ * visualization (optional) one of:
+ * table,pieChart,multibarChart,stackedAreaChart,lineChart,scatterChart
+ * colWidth (optional), e.g. 12.0
  */
 public class NewParagraphRequest {
   String title;
   String text;
   Double index;
+  HashMap< String, Object > config;
 
   public NewParagraphRequest() {
 
@@ -42,4 +48,6 @@ public class NewParagraphRequest {
   public Double getIndex() {
     return index;
   }
+
+  public HashMap< String, Object > getConfig() { return config; }
 }

zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java

@@ -114,12 +114,14 @@ public class ZeppelinServer extends Application {
        * packaged into binary package.
        */
       heliumBundleFactory = new HeliumBundleFactory(
+          conf,
           new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
           new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
           new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
           new File(conf.getRelativeDir("lib/node_modules/zeppelin-spell")));
     } else {
       heliumBundleFactory = new HeliumBundleFactory(
+          conf,
           new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
           new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")),
           new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")),
@@ -410,4 +412,3 @@ public class ZeppelinServer extends Application {
     return !new File(conf.getRelativeDir("zeppelin-web")).isDirectory();
   }
 }
-

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -193,7 +193,8 @@ public class NotebookServer extends WebSocketServlet
       }
 
       String ticket = TicketContainer.instance.getTicket(messagereceived.principal);
-      if (ticket != null && !ticket.equals(messagereceived.ticket)) {
+      if (ticket != null &&
+          (messagereceived.ticket == null || !ticket.equals(messagereceived.ticket))) {
         /* not to pollute logs, log instead of exception */
         if (StringUtils.isEmpty(messagereceived.ticket)) {
           LOG.debug("{} message: invalid ticket {} != {}", messagereceived.op,
@@ -1081,6 +1082,7 @@ public class NotebookServer extends WebSocketServlet
     if (note != null && !note.isTrash()){
       fromMessage.put("name", Folder.TRASH_FOLDER_ID + "/" + note.getName());
       renameNote(conn, userAndRoles, notebook, fromMessage, "move");
+      notebook.moveNoteToTrash(note.getId());
     }
   }
 
@@ -2138,7 +2140,7 @@ public class NotebookServer extends WebSocketServlet
     @Override
     public void onProgressUpdate(Job job, int progress) {
       notebookServer.broadcast(note.getId(),
-          new Message(OP.PROGRESS).put("id", job.getId()).put("progress", job.progress()));
+          new Message(OP.PROGRESS).put("id", job.getId()).put("progress", progress));
     }
 
     @Override

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java

@@ -19,18 +19,31 @@ package org.apache.zeppelin.socket;
 import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest;
 import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse;
 import org.eclipse.jetty.websocket.servlet.WebSocketCreator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_ALLOWED_ORIGINS;
 
 /**
  * Responsible to create the WebSockets for the NotebookServer.
  */
 public class NotebookWebSocketCreator implements WebSocketCreator {
+
+  private static final Logger LOG = LoggerFactory.getLogger(NotebookWebSocketCreator.class);
   private NotebookServer notebookServer;
 
   public NotebookWebSocketCreator(NotebookServer notebookServer) {
     this.notebookServer = notebookServer;
   }
   public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) {
-    return new NotebookSocket(request.getHttpServletRequest(), "", notebookServer);
+    String origin = request.getHeader("Origin");
+    if (notebookServer.checkOrigin(request.getHttpServletRequest(), origin)) {
+      return new NotebookSocket(request.getHttpServletRequest(), "", notebookServer);
+    } else {
+      LOG.error("Websocket request is not allowed by {} settings. Origin: {}",
+          ZEPPELIN_ALLOWED_ORIGINS, origin);
+      return null;
+    }
   }
 
 }

zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java

@@ -256,6 +256,35 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
 
   }
 
+  @Test
+  public void testRunOnSelectionCheckbox() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      String xpathToCheckbox = getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]";
+
+      createNewNote();
+
+      waitForParagraph(1, "READY");
+      setTextOfParagraph(1, "%md My selection is ${my selection=1,1|2|3}");
+      runParagraph(1);
+
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-settings']")).click();
+      collector.checkThat("'Run on selection change' checkbox will be shown under dropdown menu ",
+        driver.findElement(By.xpath(getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-click, 'turnOnAutoRun(paragraph)')]")).isDisplayed(),
+        CoreMatchers.equalTo(true));
+
+      driver.findElement(By.xpath(xpathToCheckbox)).click();
+      collector.checkThat("If 'Run on selection change' checkbox is unchecked, 'paragraph.config.runOnSelectionChange' will be false ",
+        driver.findElement(By.xpath(getParagraphXPath(1) + "//ul/li/span[contains(@ng-if, 'paragraph.config.runOnSelectionChange == false')]")).isDisplayed(),
+        CoreMatchers.equalTo(true));
+
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testRunOnSelectionButton ", e);
+    }
+  }
+
   @Test
   public void testClearOutputButton() throws Exception {
     if (!endToEndTestEnabled()) {
@@ -343,7 +372,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
           driver.findElement(By.xpath(xpathToShowTitle)).getText(),
           CoreMatchers.allOf(CoreMatchers.startsWith("Show title"), CoreMatchers.containsString("Ctrl+"),
               CoreMatchers.anyOf(CoreMatchers.containsString("Option"), CoreMatchers.containsString("Alt")),
-              CoreMatchers.containsString("+t")));
+              CoreMatchers.containsString("+T")));
 
       clickAndWait(By.xpath(xpathToShowTitle));
       collector.checkThat("After Show Title : The title field contains",
@@ -355,7 +384,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
           driver.findElement(By.xpath(xpathToHideTitle)).getText(),
           CoreMatchers.allOf(CoreMatchers.startsWith("Hide title"), CoreMatchers.containsString("Ctrl+"),
               CoreMatchers.anyOf(CoreMatchers.containsString("Option"), CoreMatchers.containsString("Alt")),
-              CoreMatchers.containsString("+t")));
+              CoreMatchers.containsString("+T")));
 
       clickAndWait(By.xpath(xpathToHideTitle));
       ZeppelinITUtils.turnOffImplicitWaits(driver);
@@ -409,7 +438,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
           driver.findElement(By.xpath(xpathToShowLineNumberButton)).getText(),
           CoreMatchers.allOf(CoreMatchers.startsWith("Show line numbers"), CoreMatchers.containsString("Ctrl+"),
               CoreMatchers.anyOf(CoreMatchers.containsString("Option"), CoreMatchers.containsString("Alt")),
-              CoreMatchers.containsString("+m")));
+              CoreMatchers.containsString("+M")));
 
 
       clickAndWait(By.xpath(xpathToShowLineNumberButton));
@@ -422,7 +451,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
           driver.findElement(By.xpath(xpathToHideLineNumberButton)).getText(),
           CoreMatchers.allOf(CoreMatchers.startsWith("Hide line numbers"), CoreMatchers.containsString("Ctrl+"),
               CoreMatchers.anyOf(CoreMatchers.containsString("Option"), CoreMatchers.containsString("Alt")),
-              CoreMatchers.containsString("+m")));
+              CoreMatchers.containsString("+M")));
 
       clickAndWait(By.xpath(xpathToHideLineNumberButton));
       collector.checkThat("After \"Hide line number\" the Line Number is Enabled",
@@ -483,4 +512,163 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
       handleException("Exception in ParagraphActionsIT while testEditOnDoubleClick ", e);
     }
   }
-}
+
+  @Test
+  public void testSingleDynamicFormTextInput() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      createNewNote();
+
+      setTextOfParagraph(1, "%spark println(\"Hello \"+z.input(\"name\", \"world\")) ");
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("Output text is equal to value specified initially",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Hello world"));
+
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//input")).clear();
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//input")).sendKeys("Zeppelin");
+
+      collector.checkThat("After new data in text input form, output should not be changed",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Hello world"));
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("Only after running the paragraph, we can see the newly updated output",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Hello Zeppelin"));
+
+      deleteTestNotebook(driver);
+
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testSingleDynamicFormTextInput  ", e);
+    }
+  }
+
+  @Test
+  public void testSingleDynamicFormSelectForm() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      createNewNote();
+
+      setTextOfParagraph(1, "%spark println(\"Howdy \"+z.select(\"names\", Seq((\"1\",\"Alice\"), " +
+              "(\"2\",\"Bob\"),(\"3\",\"stranger\"))))");
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("Output text should not display any of the options in select form",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy "));
+
+      Select dropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[1]"))));
+      dropDownMenu.selectByVisibleText("Alice");
+      collector.checkThat("After selection in drop down menu, output should display the newly selected option",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy 1"));
+
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-settings']")).click();
+      clickAndWait(By.xpath(getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]"));
+
+      Select sameDropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[1]"))));
+      sameDropDownMenu.selectByVisibleText("Bob");
+      collector.checkThat("After 'Run on selection change' checkbox is unchecked, the paragraph should not run if selecting a different option",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy 1"));
+
+      deleteTestNotebook(driver);
+
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testSingleDynamicFormSelectForm  ", e);
+    }
+  }
+
+  @Test
+  public void testSingleDynamicFormCheckboxForm() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      createNewNote();
+
+      setTextOfParagraph(1, "%spark val options = Seq((\"han\",\"Han\"), (\"leia\",\"Leia\"), " +
+              "(\"luke\",\"Luke\")); println(\"Greetings \"+z.checkbox(\"skywalkers\",options).mkString(\" and \"))");
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("Output text should display all of the options included in check boxes",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.containsString("Greetings han and leia and luke"));
+
+      WebElement firstCheckbox = driver.findElement(By.xpath("(" + getParagraphXPath(1) + "//input[@type='checkbox'])[1]"));
+      firstCheckbox.click();
+      collector.checkThat("After unchecking one of the boxes, we can see the newly updated output without the option we unchecked",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.containsString("Greetings leia and luke"));
+
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-settings']")).click();
+      clickAndWait(By.xpath(getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]"));
+
+      WebElement secondCheckbox = driver.findElement(By.xpath("(" + getParagraphXPath(1) + "//input[@type='checkbox'])[2]"));
+      secondCheckbox.click();
+      collector.checkThat("After 'Run on selection change' checkbox is unchecked, the paragraph should not run if check box state is modified",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.containsString("Greetings leia and luke"));
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+
+
+      deleteTestNotebook(driver);
+
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testSingleDynamicFormCheckboxForm  ", e);
+    }
+  }
+
+  @Test
+  public void testMultipleDynamicFormsSameType() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      createNewNote();
+
+      setTextOfParagraph(1, "%spark println(\"Howdy \"+z.select(\"fruits\", Seq((\"1\",\"Apple\")," +
+              "(\"2\",\"Orange\"),(\"3\",\"Peach\")))); println(\"Howdy \"+z.select(\"planets\", " +
+              "Seq((\"1\",\"Venus\"),(\"2\",\"Earth\"),(\"3\",\"Mars\"))))");
+
+      runParagraph(1);
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("Output text should not display any of the options in select form",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy \nHowdy "));
+
+      Select dropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[1]"))));
+      dropDownMenu.selectByVisibleText("Apple");
+      collector.checkThat("After selection in drop down menu, output should display the new option we selected",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy 1\nHowdy "));
+
+      driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-settings']")).click();
+      clickAndWait(By.xpath(getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]"));
+
+      Select sameDropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[2]"))));
+      sameDropDownMenu.selectByVisibleText("Earth");
+      waitForParagraph(1, "FINISHED");
+      collector.checkThat("After 'Run on selection change' checkbox is unchecked, the paragraph should not run if selecting a different option",
+              driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),
+              CoreMatchers.equalTo("Howdy 1\nHowdy "));
+
+      deleteTestNotebook(driver);
+
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testMultipleDynamicFormsSameType  ", e);
+    }
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java

@@ -133,8 +133,11 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
     String noteName = "test";
     String jsonRequest = "{\"name\":\"" + noteName + "\", \"paragraphs\": [" +
         "{\"title\": \"title1\", \"text\": \"text1\"}," +
-        "{\"title\": \"title2\", \"text\": \"text2\"}" +
-        "]}";
+        "{\"title\": \"title2\", \"text\": \"text2\"}," +
+        "{\"title\": \"titleConfig\", \"text\": \"text3\", " +
+        "\"config\": {\"colWidth\": 9.0, \"title\": true, "+
+        "\"results\": [{\"graph\": {\"mode\": \"pieChart\"}}] "+
+        "}}]} ";
     PostMethod post = httpPost("/notebook/", jsonRequest);
     LOG.info("testNoteCreate \n" + post.getResponseBodyAsString());
     assertThat("test note create method:", post, isAllowed());
@@ -154,13 +157,20 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
       expectedNoteName = "Note " + newNoteId;
     }
     assertEquals("compare note name", expectedNoteName, newNoteName);
-    assertEquals("initial paragraph check failed", 3, newNote.getParagraphs().size());
+    assertEquals("initial paragraph check failed", 4, newNote.getParagraphs().size());
     for (Paragraph p : newNote.getParagraphs()) {
       if (StringUtils.isEmpty(p.getText())) {
         continue;
       }
       assertTrue("paragraph title check failed", p.getTitle().startsWith("title"));
       assertTrue("paragraph text check failed", p.getText().startsWith("text"));
+      if ( p.getTitle() == "titleConfig"){
+        assertEquals("paragraph col width check failed", 9.0, p.getConfig().get("colWidth"));
+        assertTrue("paragraph show title check failed", ((boolean) p.getConfig().get("title")));
+        Map graph = ((List<Map>)p.getConfig().get("results")).get(0);
+        String mode = graph.get("mode").toString();
+        assertEquals("paragraph graph mode check failed", "pieChart", mode);
+      }
     }
     // cleanup
     ZeppelinServer.notebook.removeNote(newNoteId, anonymous);
@@ -213,8 +223,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
 
 
   @Test
-  public void testexportNote() throws IOException {
-    LOG.info("testexportNote");
+  public void testExportNote() throws IOException {
+    LOG.info("testExportNote");
     Note note = ZeppelinServer.notebook.createNote(anonymous);
     assertNotNull("can't create new note", note);
     note.setName("source note for export");
@@ -246,7 +256,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
   public void testImportNotebook() throws IOException {
     Map<String, Object> resp;
     String noteName = "source note for import";
-    LOG.info("testImortNote");
+    LOG.info("testImportNote");
     // create test note
     Note note = ZeppelinServer.notebook.createNote(anonymous);
     assertNotNull("can't create new note", note);
@@ -620,6 +630,25 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
     assertEquals("title2", paragraphAtIdx0.getTitle());
     assertEquals("text2", paragraphAtIdx0.getText());
 
+    //append paragraph providing graph
+    String jsonRequest3 = "{\"title\": \"title3\", \"text\": \"text3\", "+
+                          "\"config\": {\"colWidth\": 9.0, \"title\": true, "+
+                          "\"results\": [{\"graph\": {\"mode\": \"pieChart\"}}]}}";
+    PostMethod post3 = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest3);
+    LOG.info("testInsertParagraph response4\n" + post3.getResponseBodyAsString());
+    assertThat("Test insert method:", post3, isAllowed());
+    post3.releaseConnection();
+
+    Paragraph p = note.getLastParagraph();
+    assertEquals("title3", p.getTitle());
+    assertEquals("text3", p.getText());
+    Map result = ((List<Map>)p.getConfig().get("results")).get(0);
+    String mode = ((Map)result.get("graph")).get("mode").toString();
+    assertEquals("pieChart", mode);
+    assertEquals(9.0, p.getConfig().get("colWidth"));
+    assertTrue(((boolean) p.getConfig().get("title")));
+
+
     ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
   }
 

zeppelin-web/src/app/helium/helium.controller.js

@@ -149,37 +149,55 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
     return license;
   }
 
-  $scope.enable = function(name, artifact, type, groupId) {
+  const getHeliumTypeText = function(type) {
+    if (type === HeliumType.VISUALIZATION) {
+      return `<a target="_blank" href="https://zeppelin.apache.org/docs/${$rootScope.zeppelinVersion}/development/writingzeppelinvisualization.html">${type}</a>`; // eslint-disable-line max-len
+    } else if (type === HeliumType.SPELL) {
+      return `<a target="_blank" href="https://zeppelin.apache.org/docs/${$rootScope.zeppelinVersion}/development/writingzeppelinspell.html">${type}</a>`; // eslint-disable-line max-len
+    } else {
+      return type;
+    }
+  }
+
+  $scope.enable = function(name, artifact, type, groupId, description) {
     var license = getLicense(name, artifact);
     var mavenArtifactInfoToHTML = groupId +':'+ artifact.split('@')[0] + ':' + artifact.split('@')[1];
     var zeppelinVersion = $rootScope.zeppelinVersion;
     var url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html';
-    
+
     var confirm = ''
-    if (type === 'INTERPRETER') {
-    confirm = BootstrapDialog.show({
-      title: '',
-      message: '<p>Below command will download maven artifact ' +
-      '<code style="font-size: 11.5px; background-color: #f5f5f5; color: #0a0a0a">' +
+    if (type === HeliumType.INTERPRETER) {
+      confirm = BootstrapDialog.show({
+        title: '',
+        message: '<p>Below command will download maven artifact ' +
+        '<code style="font-size: 11.5px; background-color: #f5f5f5; color: #0a0a0a">' +
         mavenArtifactInfoToHTML + '</code>' +
-      ' and all of its transitive dependencies into interpreter/interpreter-name directory.<p>' +
-      '<div class="highlight"><pre><code class="text language-text" data-lang="text" style="font-size: 11.5px">' +
-      './bin/install-interpreter.sh --name "interpreter-name" --artifact ' +
+        ' and all of its transitive dependencies into interpreter/interpreter-name directory.<p>' +
+        '<div class="highlight"><pre><code class="text language-text" data-lang="text" style="font-size: 11.5px">' +
+        './bin/install-interpreter.sh --name "interpreter-name" --artifact ' +
         mavenArtifactInfoToHTML +' </code></pre>' +
-      '<p>After restart Zeppelin, create interpreter setting and bind it with your note. ' +
-      'For more detailed information, see <a target="_blank" href=' +
+        '<p>After restart Zeppelin, create interpreter setting and bind it with your note. ' +
+        'For more detailed information, see <a target="_blank" href=' +
         url + '>Interpreter Installation.</a></p>'
-    });
+      });
     } else {
       confirm = BootstrapDialog.confirm({
         closable: false,
         closeByBackdrop: false,
         closeByKeyboard: false,
-        title: '',
-        message: 'Do you want to enable ' + name + '?' +
-        '<div style="color:gray">' + artifact + '</div>' +
-        '<div style="border-top: 1px solid #efefef; margin-top: 10px; padding-top: 5px;">License</div>' +
-        '<div style="color:gray">' + license + '</div>',
+        title: '<div style="font-weight: 300;">Do you want to enable Helium Package?</div>',
+        message:
+          '<div style="font-size: 14px; margin-top: 5px;">Artifact</div>' +
+          `<div style="color:gray">${artifact}</div>` +
+          '<hr style="margin-top: 10px; margin-bottom: 10px;" />' +
+          '<div style="font-size: 14px; margin-bottom: 2px;">Type</div>' +
+          `<div style="color:gray">${getHeliumTypeText(type)}</div>` +
+          '<hr style="margin-top: 10px; margin-bottom: 10px;" />' +
+          '<div style="font-size: 14px;">Description</div>' +
+          `<div style="color:gray">${description}</div>` +
+          '<hr style="margin-top: 10px; margin-bottom: 10px;" />' +
+          '<div style="font-size: 14px;">License</div>' +
+          `<div style="color:gray">${license}</div>`,
         callback: function (result) {
           if (result) {
             confirm.$modalFooter.find('button').addClass('disabled');
@@ -203,13 +221,13 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
     }
   };
 
-  $scope.disable = function(name) {
+  $scope.disable = function(name, artifact) {
     var confirm = BootstrapDialog.confirm({
       closable: false,
       closeByBackdrop: false,
       closeByKeyboard: false,
-      title: '',
-      message: 'Do you want to disable ' + name + '?',
+      title: '<div style="font-weight: 300;">Do you want to disable Helium Package?</div>',
+      message: artifact,
       callback: function(result) {
         if (result) {
           confirm.$modalFooter.find('button').addClass('disabled');
@@ -309,5 +327,9 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
     });
   };
 
+  $scope.getDescriptionText = function(pkgSearchResult) {
+    return $sce.trustAsHtml(pkgSearchResult.pkg.description);
+  };
+
   init();
 }

zeppelin-web/src/app/helium/helium.css

@@ -12,6 +12,10 @@
  * limitations under the License.
  */
 
+.heliumHead {
+  padding-bottom: 34px !important;
+}
+
 .heliumPackageContainer {
   padding-bottom: 0px;
   margin-bottom: 0px;
@@ -113,6 +117,10 @@
   float: left;
 }
 
+.heliumBundleOrder p {
+  margin:0 0 5px 15px
+}
+
 .sortable-row .as-sortable-item-handle {
   width: 35px;
   height: 30px;
@@ -162,6 +170,10 @@
 }
 
 .localPkgInfo {
+  margin-right: 0;
+}
+
+.localPkgInfo p {
   margin: 10px 12px 0 0;
   font-size: 11px;
   font-style: italic;

zeppelin-web/src/app/helium/helium.html

@@ -11,7 +11,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->
-<div class="interpreterHead">
+<div class="interpreterHead"
+     ng-class="(bundleOrder.length > 1) ? '' : 'heliumHead'">
   <div class="header">
     <div class="row">
       <div class="col-md-12">
@@ -34,28 +35,34 @@ limitations under the License.
             <i class="fa fa-cube"></i>
             {{pkgTypes}}
           </button>
-          <p class="localPkgInfo">* Local registry package's name is gray colored.</p>
         </div>
       </div>
     </div>
     <div ng-show="bundleOrder.length > 1"
          class="row heliumBundleOrder">
-      <div style="margin:0 0 5px 15px">Bundle package display order (drag and drop to reorder)</div>
-      <div class="col-md-12 sortable-row btn-group"
-           as-sortable="bundleOrderListeners"
-           data-ng-model="bundleOrder">
-        <div class="btn-group" data-ng-repeat="pkgName in bundleOrder"
-             as-sortable-item>
-          <div class="btn btn-default btn-sm"
-               ng-bind-html='defaultPackages[pkgName].pkg.icon'
-               as-sortable-item-handle>
+      <p>Bundle package display order (drag and drop to reorder)</p>
+      <div class="col-md-12">
+        <div class="sortable-row btn-group"
+             as-sortable="bundleOrderListeners"
+             data-ng-model="bundleOrder">
+          <div class="btn-group" data-ng-repeat="pkgName in bundleOrder"
+               as-sortable-item>
+            <div class="btn btn-default btn-sm"
+                 ng-bind-html='defaultPackages[pkgName].pkg.icon'
+                 as-sortable-item-handle>
+            </div>
           </div>
         </div>
-        <span class="saveLink"
-              ng-show="bundleOrderChanged"
-              ng-click="saveBundleOrder()">
+        <div class="saveLink"
+             ng-show="bundleOrderChanged"
+             ng-click="saveBundleOrder()">
           save
-        </span>
+        </div>
+      </div>
+    </div>
+    <div class="pull-right row localPkgInfo">
+      <div class="col-md-12">
+        <p>* Local registry package's name is gray colored.</p>
       </div>
     </div>
   </div>
@@ -96,11 +103,11 @@ limitations under the License.
           <span class="heliumType">{{pkgSearchResult.pkg.type}}</span>
         </div>
         <div ng-show="!pkgSearchResult.enabled"
-             ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId)"
+             ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId, pkgSearchResult.pkg.description)"
              class="btn btn-success btn-xs"
              style="float:right">Enable</div>
         <div ng-show="pkgSearchResult.enabled"
-             ng-click="disable(pkgSearchResult.pkg.name)"
+             ng-click="disable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact)"
              ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
              class="btn btn-info btn-xs"
              style="float:right">Disable</div>
@@ -135,9 +142,7 @@ limitations under the License.
           </a>
         </li>
       </ul>
-      <div class="heliumPackageDescription">
-        {{pkgSearchResult.pkg.description}}
-      </div>
+      <div class="heliumPackageDescription" ng-bind-html="getDescriptionText(pkgSearchResult)" />
       <div ng-if="pkgSearchResult.pkg.type === 'SPELL' && pkgSearchResult.pkg.spell"
            class="spellInfo">
         <div>

zeppelin-web/src/app/home/home.css

@@ -298,10 +298,21 @@ a.navbar-brand:hover {
 }
 
 @media (min-width: 768px) {
-  .dropdown-menu.navbar-dropdown-maxHeight {
+  .navbar-fixed-top .dropdown-menu {
     max-height: calc(100vh - 60px);
     overflow: auto;
   }
+  #actionbar .dropdown-menu {
+    max-height: calc(100vh - 110px);
+    overflow: auto;
+  }
+}
+
+@media (max-width: 767px) {
+  #actionbar .dropdown-menu {
+    max-height: calc(100vh - 160px);
+    overflow: auto;
+  }
 }
 
 .nav-component {

zeppelin-web/src/app/notebook/paragraph/paragraph-control.html

@@ -38,7 +38,7 @@ limitations under the License.
     {{paragraph.status}}
   </span>
 
-  <span ng-if="paragraph.status=='RUNNING'">
+  <span ng-if="paragraph.status === 'RUNNING' && paragraph.executor !== 'SPELL'">
     {{getProgress()}}%
   </span>
 
@@ -48,7 +48,7 @@ limitations under the License.
           ng-click="runParagraphFromButton(getEditorValue())"
           ng-show="paragraph.status!='RUNNING' && paragraph.status!='PENDING' && paragraph.config.enabled"></span>
     <span class="icon-control-pause" style="cursor:pointer;color:#CD5C5C" tooltip-placement="top"
-          tooltip="Cancel (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+c)"
+          tooltip="Cancel (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+C)"
           ng-click="cancelParagraph(paragraph)"
           ng-show="paragraph.status=='RUNNING' || paragraph.status=='PENDING'"></span>
     <span ng-show="paragraph.runtimeInfos.jobUrl.length == 1">
@@ -66,10 +66,10 @@ limitations under the License.
       </ul>
     </span>
     <span class="{{paragraph.config.editorHide ? 'icon-size-fullscreen' : 'icon-size-actual'}}" style="cursor:pointer" tooltip-placement="top"
-          tooltip="{{(paragraph.config.editorHide ? 'Show' : 'Hide')}} editor (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+e)"
+          tooltip="{{(paragraph.config.editorHide ? 'Show' : 'Hide')}} editor (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+E)"
           ng-click="toggleEditor(paragraph)"></span>
     <span class="{{paragraph.config.tableHide ? 'icon-notebook' : 'icon-book-open'}}" style="cursor:pointer" tooltip-placement="top"
-          tooltip="{{(paragraph.config.tableHide ? 'Show' : 'Hide')}} output (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+o)"
+          tooltip="{{(paragraph.config.tableHide ? 'Show' : 'Hide')}} output (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+O)"
           ng-click="toggleOutput(paragraph)"></span>
     <span class="dropdown navbar-right">
       <span class="icon-settings" style="cursor:pointer"
@@ -89,6 +89,18 @@ limitations under the License.
           </a>
         </li>
         <li role="separator" class="divider"></li>
+        <li style="padding-top:8px"
+            ng-if="paragraph.config.runOnSelectionChange == true || paragraph.config.runOnSelectionChange == false">
+          <span ng-if="paragraph.config.runOnSelectionChange == true" class="fa fa-toggle-on shortcut-icon" style="padding:3px 8px 8px 20px"></span>
+          <span ng-if="paragraph.config.runOnSelectionChange == false" class="fa fa-toggle-off shortcut-icon" style="padding:3px 8px 8px 20px"></span>Run on selection change
+          <form style="display:inline; float:right">
+            <input type="checkbox"
+                   style="width:16px; margin-right:20px"
+                   tooltip-placement="top" tooltip="Even if you uncheck this, still can run the paragraph by pressing Enter"
+                   ng-checked="{{paragraph.config.runOnSelectionChange}}"
+                   ng-click="turnOnAutoRun(paragraph)"/>
+          </form>
+        </li>
         <li>
           <a ng-click="$event.stopPropagation()" class="dropdown"><span class="fa fa-arrows-h shortcut-icon"></span>Width
             <form style="display:inline; margin-left:5px; float:right">
@@ -100,55 +112,55 @@ limitations under the License.
           </a>
         </li>
         <li>
-          <a ng-click="moveUp(paragraph)" ng-hide="$first"><span class="icon-arrow-up shortcut-icon"></span>Move Up
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+k</span></a>
+          <a ng-click="moveUp(paragraph)" ng-hide="$first"><span class="icon-arrow-up shortcut-icon"></span>Move up
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+K</span></a>
         </li>
         <li>
-          <a ng-click="moveDown(paragraph)" ng-hide="$last"><span class="icon-arrow-down shortcut-icon"></span>Move Down
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+j</span></a>
+          <a ng-click="moveDown(paragraph)" ng-hide="$last"><span class="icon-arrow-down shortcut-icon"></span>Move down
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+J</span></a>
         </li>
         <li>
-          <a ng-click="insertNew('below')"><span class="icon-plus shortcut-icon"></span>Insert New
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+b</span></a>
+          <a ng-click="insertNew('below')"><span class="icon-plus shortcut-icon"></span>Insert new
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+B</span></a>
         </li>
         <li>
           <a ng-click="copyParagraph(getEditorValue())"><span class="fa fa-copy shortcut-icon"></span>Clone paragraph
-            <span class="shortcut-keys">Ctrl+Shift+c</span></a>
+            <span class="shortcut-keys">Ctrl+Shift+C</span></a>
         </li>
         <li>
           <!-- paragraph handler -->
           <a ng-click="hideTitle(paragraph)"
              ng-show="paragraph.config.title"><span class="fa fa-font shortcut-icon"></span>Hide title
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+t</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+T</span></a>
           <a ng-click="showTitle(paragraph)"
              ng-show="!paragraph.config.title"><span class="fa fa-font shortcut-icon"></span>Show title
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+t</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+T</span></a>
         </li>
         <li>
           <a ng-click="hideLineNumbers(paragraph)"
              ng-show="paragraph.config.lineNumbers"><span class="fa fa-list-ol shortcut-icon"></span>Hide line numbers
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+m</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+M</span></a>
           <a ng-click="showLineNumbers(paragraph)"
              ng-show="!paragraph.config.lineNumbers"><span class="fa fa-list-ol shortcut-icon"></span>Show line numbers
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+m</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+M</span></a>
         </li>
         <li>
           <a ng-click="toggleEnableDisable(paragraph)"><span class="icon-control-play shortcut-icon"></span>
             {{paragraph.config.enabled ? "Disable" : "Enable"}} run
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+r</span></a>
+            <span class="shortcut-keys">Ctrl+ {{ isMac ? 'Option' : 'Alt'}}+R</span></a>
         </li>
         <li>
           <a ng-click="goToSingleParagraph()"><span class="icon-share-alt shortcut-icon"></span>Link this paragraph
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+w</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+W</span></a>
         </li>
         <li>
           <a ng-click="clearParagraphOutput(paragraph)"><span class="fa fa-eraser shortcut-icon"></span>Clear output
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+l</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+L</span></a>
         </li>
         <li>
           <!-- remove paragraph -->
           <a ng-click="removeParagraph(paragraph)" ng-hide="$last"><span class="fa fa-times shortcut-icon"></span>Remove
-            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+d</span></a>
+            <span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+D</span></a>
         </li>
       </ul>
     </span>

zeppelin-web/src/app/notebook/paragraph/paragraph-parameterizedQueryForm.html

@@ -19,33 +19,53 @@ limitations under the License.
        ng-init="loadForm(formulaire, paragraph.settings.params)">
     <label class="control-label input-sm" ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }">{{formulaire.name}}</label>
     <div>
-
       <input class="form-control input-sm"
              ng-if="!paragraph.settings.forms[formulaire.name].options"
              ng-enter="runParagraphFromButton(getEditorValue())"
              ng-model="paragraph.settings.params[formulaire.name]"
              ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
              name="{{formulaire.name}}" />
-
+    </div>
+    <div ng-if="paragraph.config.runOnSelectionChange == true">
       <select class="form-control input-sm"
              ng-if="paragraph.settings.forms[formulaire.name].options && paragraph.settings.forms[formulaire.name].type != 'checkbox'"
-             ng-enter="runParagraphFromButton(getEditorValue())"
+             ng-change="runParagraphFromButton(getEditorValue())"
              ng-model="paragraph.settings.params[formulaire.name]"
              ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
              name="{{formulaire.name}}"
              ng-options="option.value as (option.displayName||option.value) for option in paragraph.settings.forms[formulaire.name].options">
       </select>
-
-      <div ng-if="paragraph.settings.forms[formulaire.name].type == 'checkbox'">
-        <label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
-               class="checkbox-item input-sm">
-          <input type="checkbox"
-                 ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
-                 ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
-                 ng-click="toggleCheckbox(formulaire, option, false)"/>{{option.displayName||option.value}}
-        </label>
-      </div>
-
+    </div>
+    <div ng-if="paragraph.config.runOnSelectionChange == false">
+      <select class="form-control input-sm"
+              ng-if="paragraph.settings.forms[formulaire.name].options && paragraph.settings.forms[formulaire.name].type != 'checkbox'"
+              ng-enter="runParagraphFromButton(getEditorValue())"
+              ng-model="paragraph.settings.params[formulaire.name]"
+              ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
+              name="{{formulaire.name}}"
+              ng-options="option.value as (option.displayName||option.value) for option in paragraph.settings.forms[formulaire.name].options">
+      </select>
+    </div>
+    <div ng-if="paragraph.config.runOnSelectionChange == true &&
+                paragraph.settings.forms[formulaire.name].type == 'checkbox'">
+      <label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
+             class="checkbox-item input-sm">
+        <input type="checkbox"
+               ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
+               ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
+               ng-click="toggleCheckbox(formulaire, option, false); runParagraphFromButton(getEditorValue())"/> {{option.displayName||option.value}}
+      </label>
+    </div>
+    <div ng-if="paragraph.config.runOnSelectionChange == false &&
+                paragraph.settings.forms[formulaire.name].type == 'checkbox'">
+      <label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
+             class="checkbox-item input-sm">
+        <input type="checkbox"
+               ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
+               ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
+               ng-enter="runParagraphFromButton(getEditorValue())"
+               ng-click="toggleCheckbox(formulaire, option, false)"/> {{option.displayName||option.value}}
+      </label>
     </div>
   </div>
 </form>

zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js

@@ -17,6 +17,12 @@ import {
   ParagraphStatus, isParagraphRunning,
 } from './paragraph.status';
 
+const ParagraphExecutor = {
+  SPELL: 'SPELL',
+  INTERPRETER: 'INTERPRETER',
+  NONE: '', /** meaning `DONE` */
+};
+
 angular.module('zeppelinWebApp').controller('ParagraphCtrl', ParagraphCtrl);
 
 function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $location,
@@ -26,6 +32,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
   'ngInject';
 
   var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_';
+  $rootScope.keys = Object.keys;
   $scope.parentNote = null;
   $scope.paragraph = {};
   $scope.paragraph.results = {};
@@ -121,13 +128,25 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
   };
 
   var initializeDefault = function(config) {
+    var forms = $scope.paragraph.settings.forms;
+    
     if (!config.colWidth) {
       config.colWidth = 12;
     }
-
+  
     if (config.enabled === undefined) {
       config.enabled = true;
     }
+  
+    for (var idx in forms) {
+      if (forms[idx]) {
+        if (forms[idx].options) {
+          if (config.runOnSelectionChange === undefined) {
+            config.runOnSelectionChange = true;
+          }
+        }
+      }
+    }
 
     if (!config.results) {
       config.results = {};
@@ -266,6 +285,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
 
   $scope.cleanupSpellTransaction = function() {
     const status = ParagraphStatus.FINISHED;
+    $scope.paragraph.executor = ParagraphExecutor.NONE;
     $scope.paragraph.status = status;
     $scope.paragraph.results.code = status;
 
@@ -284,8 +304,9 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
 
   $scope.runParagraphUsingSpell = function(paragraphText,
                                            magic, digestRequired, propagated) {
+    $scope.paragraph.status = 'RUNNING';
+    $scope.paragraph.executor = ParagraphExecutor.SPELL;
     $scope.paragraph.results = {};
-    $scope.paragraph.status = ParagraphStatus.RUNNING;
     $scope.paragraph.errorMessage = '';
     if (digestRequired) { $scope.$digest(); }
 
@@ -343,7 +364,6 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
     if (!paragraphText || $scope.isRunning($scope.paragraph)) {
       return;
     }
-
     const magic = SpellResult.extractMagic(paragraphText);
 
     if (heliumService.getSpellByMagic(magic)) {
@@ -377,6 +397,11 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
     $scope.runParagraph(paragraphText, false, false)
   };
 
+  $scope.turnOnAutoRun = function (paragraph) {
+    paragraph.config.runOnSelectionChange = !paragraph.config.runOnSelectionChange;
+    commitParagraph(paragraph);
+  };
+
   $scope.moveUp = function(paragraph) {
     $scope.$emit('moveParagraphUp', paragraph);
   };
@@ -537,7 +562,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
     if ($scope.asIframe) {
       return 'col-md-12';
     } else {
-      return 'col-md-' + n;
+      return 'paragraph-col col-md-' + n;
     }
   };
 

zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js

@@ -5,7 +5,10 @@ describe('Controller: ParagraphCtrl', function() {
   var scope;
   var websocketMsgSrvMock = {};
   var paragraphMock = {
-    config: {}
+    config: {},
+    settings: {
+      forms: {}
+    }
   };
   var route = {
     current: {

zeppelin-web/src/app/visualization/builtins/visualization-barchart.js

@@ -41,7 +41,7 @@ export default class BarchartVisualization extends Nvd3ChartVisualization {
       pivot.groups,
       pivot.values,
       true,
-      false,
+      true,
       true);
 
     super.render(d3Data);

zeppelin-web/src/app/visualization/builtins/visualization-linechart.js

@@ -70,7 +70,12 @@ export default class LinechartVisualization extends Nvd3ChartVisualization {
   configureChart(chart) {
     var self = this;
     chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
-    chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d, self.xLabels);});
+    chart.yAxis.tickFormat(function(d) {
+      if (d === undefined) {
+        return 'N/A';
+      }
+      return self.yAxisTickFormat(d, self.xLabels);
+    });
     chart.yAxis.axisLabelDistance(50);
     if (chart.useInteractiveGuideline) {   // lineWithFocusChart hasn't got useInteractiveGuideline
       chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)
@@ -111,4 +116,8 @@ export default class LinechartVisualization extends Nvd3ChartVisualization {
       }
     };
   };
+
+  defaultY() {
+    return undefined;
+  };
 }

zeppelin-web/src/app/visualization/builtins/visualization-nvd3chart.js

@@ -81,6 +81,10 @@ export default class Nvd3ChartVisualization extends Visualization {
     return s;
   };
 
+  defaultY() {
+    return 0;
+  };
+
   xAxisTickFormat(d, xLabels) {
     if (xLabels[d] && (isNaN(parseFloat(xLabels[d])) || !isFinite(xLabels[d]))) { // to handle string type xlabel
       return xLabels[d];
@@ -98,6 +102,7 @@ export default class Nvd3ChartVisualization extends Visualization {
 
   d3DataFromPivot(
     schema, rows, keys, groups, values, allowTextXAxis, fillMissingValues, multiBarChart) {
+    var self = this;
     // construct table data
     var d3g = [];
 
@@ -181,10 +186,10 @@ export default class Nvd3ChartVisualization extends Visualization {
         }
 
         var xVar = isNaN(rowValue) ? ((allowTextXAxis) ? rowValue : rowNameIndex[rowValue]) : parseFloat(rowValue);
-        var yVar = 0;
+        var yVar = self.defaultY();
         if (xVar === undefined) { xVar = colName; }
         if (value !== undefined) {
-          yVar = isNaN(value.value) ? 0 : parseFloat(value.value) / parseFloat(value.count);
+          yVar = isNaN(value.value) ? self.defaultY() : parseFloat(value.value) / parseFloat(value.count);
         }
         d3g[i].values.push({
           x: xVar,

zeppelin-web/src/app/visualization/builtins/visualization-piechart.js

@@ -33,7 +33,9 @@ export default class PiechartVisualization extends Nvd3ChartVisualization {
   };
 
   render(pivot) {
-    var d3Data = this.d3DataFromPivot(
+    // [ZEPPELIN-2253] New chart function will be created each time inside super.render()
+    this.chart = null;
+    const d3Data = this.d3DataFromPivot(
       pivot.schema,
       pivot.rows,
       pivot.keys,
@@ -42,17 +44,25 @@ export default class PiechartVisualization extends Nvd3ChartVisualization {
       true,
       false,
       false);
-    var d = d3Data.d3g;
-    var d3g = [];
-    if (d.length > 0) {
-      for (var i = 0; i < d[0].values.length ; i++) {
-        var e = d[0].values[i];
-        d3g.push({
-          label: e.x,
-          value: e.y
-        });
-      }
+    const d = d3Data.d3g;
+
+    let generateLabel;
+    // data is grouped
+    if (pivot.groups && pivot.groups.length > 0) {
+      generateLabel = (suffix, prefix) => `${prefix}.${suffix}`;
+    } else { // data isn't grouped
+      generateLabel = suffix => suffix;
     }
+
+    let d3g = d.map(group => {
+      return group.values.map(row => ({
+        label: generateLabel(row.x, group.key),
+        value: row.y
+      }));
+    });
+    // the map function returns d3g as a nested array
+    // [].concat flattens it, http://stackoverflow.com/a/10865042/5154397
+    d3g = [].concat.apply([], d3g);
     super.render({d3g: d3g});
   };
 

zeppelin-web/src/components/modal-shortcut/modal-shortcut.html

@@ -37,7 +37,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">c</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">C</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -48,7 +48,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">p</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">P</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -59,7 +59,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">n</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">N</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -70,7 +70,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">d</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">D</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -81,7 +81,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">a</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">A</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -92,7 +92,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">b</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">B</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -103,7 +103,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">Shift</kbd> + <kbd class="kbd-dark">c</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">Shift</kbd> + <kbd class="kbd-dark">C</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -114,7 +114,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">k</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">K</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -125,7 +125,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">j</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">J</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -136,7 +136,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt' }}</kbd> + <kbd class="kbd-dark">r</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt' }}</kbd> + <kbd class="kbd-dark">R</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -147,7 +147,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">o</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">O</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -158,7 +158,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">e</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">E</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -169,7 +169,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">m</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">M</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -180,7 +180,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">t</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">T</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -191,7 +191,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">l</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">L</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -202,7 +202,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">w</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">{{ isMac ? 'Option' : 'Alt'}}</kbd> + <kbd class="kbd-dark">W</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -250,7 +250,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">k</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">K</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -261,7 +261,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">y</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">Y</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -272,7 +272,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">s</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">S</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -283,7 +283,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">a</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">A</kbd>
             </div>
           </div>
           <div class="col-md-8">
@@ -294,7 +294,7 @@ limitations under the License.
         <div class="row">
           <div class="col-md-4">
             <div class="keys">
-              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">e</kbd>
+              <kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">E</kbd>
             </div>
           </div>
           <div class="col-md-8">

zeppelin-web/src/components/navbar/navbar.html

@@ -27,7 +27,7 @@ limitations under the License.
       <ul class="nav navbar-nav" ng-if="ticket">
         <li class="dropdown notebook-list-dropdown" dropdown>
           <a href="#" class="dropdown-toggle" data-toggle="dropdown" dropdown-toggle>Notebook <span class="caret"></span></a>
-          <ul class="dropdown-menu navbar-dropdown-maxHeight" role="menu">
+          <ul class="dropdown-menu" role="menu">
             <li ng-controller="NotenameCtrl as notenamectrl"><a href="" data-toggle="modal" data-target="#noteNameModal" ng-click="notenamectrl.getInterpreterSettings()"><i class="fa fa-plus"></i> Create new note</a></li>
             <li class="divider"></li>
             <div id="notebook-list" class="scrollbar-container" ng-if="isDrawNavbarNoteList">

zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -108,7 +108,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
         conf = new ZeppelinConfiguration();
       }
     }
-    
+
     LOG.info("Server Host: " + conf.getServerAddress());
     if (conf.useSsl() == false) {
       LOG.info("Server Port: " + conf.getServerPort());
@@ -355,7 +355,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   public String getNotebookDir() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
   }
-  
+
   public String getUser() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER);
   }
@@ -363,7 +363,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   public String getBucketName() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
   }
-  
+
   public String getEndpoint() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
   }
@@ -375,7 +375,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   public String getS3KMSKeyRegion() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
   }
-  
+
   public String getS3EncryptionMaterialsProviderClass() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
   }
@@ -424,6 +424,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
   }
 
+  public String getHeliumNpmRegistry() {
+    return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY);
+  }
+
   public String getNotebookAuthorizationPath() {
     return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
   }
@@ -449,6 +453,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
   }
 
+  public String getInterpreterMvnRepoPath() {
+    return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO);
+  }
+
   public String getRelativeDir(ConfVars c) {
     return getRelativeDir(getString(c));
   }
@@ -464,7 +472,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   public boolean isWindowsPath(String path){
     return path.matches("^[A-Za-z]:\\\\.*");
   }
-  
+
   public boolean isAnonymousAllowed() {
     return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
   }
@@ -472,7 +480,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   public boolean isNotebokPublic() {
     return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC);
   }
-  
+
   public String getConfDir() {
     return getString(ConfVars.ZEPPELIN_CONF_DIR);
   }
@@ -591,6 +599,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
     ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
     ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
+    ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
+        "http://repo1.maven.org/maven2/"),
     ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
     ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
     ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
@@ -629,6 +639,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
     ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
     ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
+    ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/"),
     // Allows a way to specify a ',' separated list of allowed origins for rest and websockets
     // i.e. http://localhost:8080
     ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java

@@ -33,6 +33,8 @@ import java.io.*;
 import java.net.URL;
 import java.util.*;
 
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
 /**
  * Load helium visualization & spell
  */
@@ -40,7 +42,6 @@ public class HeliumBundleFactory {
   Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
   private final String NODE_VERSION = "v6.9.1";
   private final String NPM_VERSION = "3.10.8";
-  private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";
   public static final String HELIUM_LOCAL_REPO = "helium-bundle";
   public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
   public static final String HELIUM_BUNDLE = "helium.bundle.js";
@@ -52,10 +53,13 @@ public class HeliumBundleFactory {
 
   private final FrontendPluginFactory frontEndPluginFactory;
   private final File workingDirectory;
+  private ZeppelinConfiguration conf;
   private File tabledataModulePath;
   private File visualizationModulePath;
   private File spellModulePath;
+  private String defaultNpmRegistryUrl;
   private Gson gson;
+  private boolean nodeAndNpmInstalled = false;
 
   String bundleCacheKey = "";
   File currentCacheBundle;
@@ -63,18 +67,23 @@ public class HeliumBundleFactory {
   ByteArrayOutputStream out  = new ByteArrayOutputStream();
 
   public HeliumBundleFactory(
+      ZeppelinConfiguration conf,
       File moduleDownloadPath,
       File tabledataModulePath,
       File visualizationModulePath,
       File spellModulePath) throws TaskRunnerException {
-    this(moduleDownloadPath);
+    this(conf, moduleDownloadPath);
     this.tabledataModulePath = tabledataModulePath;
     this.visualizationModulePath = visualizationModulePath;
     this.spellModulePath = spellModulePath;
   }
 
-  public HeliumBundleFactory(File moduleDownloadPath) throws TaskRunnerException {
+  public HeliumBundleFactory(
+      ZeppelinConfiguration conf,
+      File moduleDownloadPath) throws TaskRunnerException {
     this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+    this.conf = conf;
+    this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
     File installDirectory = workingDirectory;
 
     frontEndPluginFactory = new FrontendPluginFactory(
@@ -82,11 +91,12 @@ public class HeliumBundleFactory {
 
     currentCacheBundle = new File(workingDirectory, HELIUM_BUNDLE_CACHE);
     gson = new Gson();
-    installNodeAndNpm();
-    configureLogger();
   }
 
-  private void installNodeAndNpm() {
+  void installNodeAndNpm() {
+    if (nodeAndNpmInstalled) {
+      return;
+    }
     try {
       NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
       npmInstaller.setNpmVersion(NPM_VERSION);
@@ -95,6 +105,8 @@ public class HeliumBundleFactory {
       NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
       nodeInstaller.setNodeVersion(NODE_VERSION);
       nodeInstaller.install();
+      configureLogger();
+      nodeAndNpmInstalled = true;
     } catch (InstallationException e) {
       logger.error(e.getMessage(), e);
     }
@@ -111,6 +123,20 @@ public class HeliumBundleFactory {
 
   public synchronized File buildBundle(List<HeliumPackage> pkgs, boolean forceRefresh)
       throws IOException {
+
+    if (pkgs == null || pkgs.size() == 0) {
+      // when no package is selected, simply return an empty file instead of try bundle package
+      synchronized (this) {
+        currentCacheBundle.getParentFile().mkdirs();
+        currentCacheBundle.delete();
+        currentCacheBundle.createNewFile();
+        bundleCacheKey = "";
+        return currentCacheBundle;
+      }
+    }
+
+    installNodeAndNpm();
+
     // package.json
     URL pkgUrl = Resources.getResource("helium/package.json");
     String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8);
@@ -380,7 +406,7 @@ public class HeliumBundleFactory {
     }
   }
 
-  public synchronized void install(HeliumPackage pkg) throws TaskRunnerException {
+  synchronized void install(HeliumPackage pkg) throws TaskRunnerException {
     String commandForNpmInstallArtifact =
         String.format("install %s --fetch-retries=%d --fetch-retry-factor=%d " +
                         "--fetch-retry-mintimeout=%d", pkg.getArtifact(),
@@ -393,8 +419,8 @@ public class HeliumBundleFactory {
   }
 
   private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
-    NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL);
-
+    installNodeAndNpm();
+    NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
     npm.execute(args, env);
   }
 

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java

@@ -302,6 +302,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
     String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
     int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
     String interpreterRunnerPath;
+    String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
     if (null != interpreterRunner) {
       interpreterRunnerPath = interpreterRunner.getPath();
       Path p = Paths.get(interpreterRunnerPath);
@@ -317,7 +318,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
         new RemoteInterpreter(property, interpreterSessionKey, className,
             interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
             remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
-            conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT));
+            conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
     remoteInterpreter.addEnv(env);
 
     return new LazyOpenInterpreter(remoteInterpreter);

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java

@@ -70,7 +70,7 @@ public class InterpreterSetting {
   @SerializedName("interpreterGroup")
   private List<InterpreterInfo> interpreterInfos;
   private final transient Map<String, InterpreterGroup> interpreterGroupRef = new HashMap<>();
-  private List<Dependency> dependencies;
+  private List<Dependency> dependencies = new LinkedList<>();
   private InterpreterOption option;
   private transient String path;
 
@@ -177,7 +177,7 @@ public class InterpreterSetting {
     }
   }
 
-  private String getInterpreterSessionKey(String user, String noteId) {
+  String getInterpreterSessionKey(String user, String noteId) {
     InterpreterOption option = getOption();
     String key;
     if (option.isExistingProcess()) {
@@ -226,76 +226,47 @@ public class InterpreterSetting {
     }
   }
 
-  void closeAndRemoveInterpreterGroupByNoteId(String noteId) {
-    String processKey = getInterpreterProcessKey("", noteId);
-    List<InterpreterGroup> closeToGroupList = new LinkedList<>();
-    InterpreterGroup groupKey;
-    for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
-      if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
-        interpreterGroupWriteLock.lock();
-        groupKey = interpreterGroupRef.remove(intpKey);
-        interpreterGroupWriteLock.unlock();
-        closeToGroupList.add(groupKey);
-      }
-    }
-
-    for (InterpreterGroup groupToRemove : closeToGroupList) {
-      groupToRemove.close();
-    }
-  }
-
-  void closeAndRemoveInterpreterGroupByUser(String user) {
+  void closeAndRemoveInterpreterGroup(String noteId, String user) {
     if (user.equals("anonymous")) {
       user = "";
     }
-    String processKey = getInterpreterProcessKey(user, "");
-    String sessionKey = getInterpreterSessionKey(user, "");
+    String processKey = getInterpreterProcessKey(user, noteId);
+    String sessionKey = getInterpreterSessionKey(user, noteId);
     List<InterpreterGroup> groupToRemove = new LinkedList<>();
     InterpreterGroup groupItem;
     for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
       if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
         interpreterGroupWriteLock.lock();
-        groupItem = interpreterGroupRef.remove(intpKey);
+        // TODO(jl): interpreterGroup has two or more sessionKeys inside it. thus we should not
+        // remove interpreterGroup if it has two or more values.
+        groupItem = interpreterGroupRef.get(intpKey);
         interpreterGroupWriteLock.unlock();
         groupToRemove.add(groupItem);
       }
+      for (InterpreterGroup groupToClose : groupToRemove) {
+        // TODO(jl): Fix the logic removing session. Now, it's handled into groupToClose.clsose()
+        groupToClose.close(interpreterGroupRef, intpKey, sessionKey);
+      }
+      groupToRemove.clear();
     }
 
-    for (InterpreterGroup groupToClose : groupToRemove) {
-      groupToClose.close(sessionKey);
-    }
+    //Remove session because all interpreters in this session are closed
+    //TODO(jl): Change all code to handle interpreter one by one or all at once
+
   }
 
   void closeAndRemoveAllInterpreterGroups() {
-    HashSet<String> groupsToRemove = new HashSet<>(interpreterGroupRef.keySet());
-    for (String key : groupsToRemove) {
-      closeAndRemoveInterpreterGroupByNoteId(key);
-    }
-  }
-
-  void shutdownAndRemoveInterpreterGroup(String interpreterGroupKey) {
-    String key = getInterpreterProcessKey("", interpreterGroupKey);
-
-    List<InterpreterGroup> groupToRemove = new LinkedList<>();
-    InterpreterGroup groupItem;
-    for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
-      if (isEqualInterpreterKeyProcessKey(intpKey, key)) {
-        interpreterGroupWriteLock.lock();
-        groupItem = interpreterGroupRef.remove(intpKey);
-        interpreterGroupWriteLock.unlock();
-        groupToRemove.add(groupItem);
+    for (String processKey : new HashSet<>(interpreterGroupRef.keySet())) {
+      InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
+      for (String sessionKey : new HashSet<>(interpreterGroup.keySet())) {
+        interpreterGroup.close(interpreterGroupRef, processKey, sessionKey);
       }
     }
-
-    for (InterpreterGroup groupToClose : groupToRemove) {
-      groupToClose.shutdown();
-    }
   }
 
   void shutdownAndRemoveAllInterpreterGroups() {
-    HashSet<String> groupsToRemove = new HashSet<>(interpreterGroupRef.keySet());
-    for (String interpreterGroupKey : groupsToRemove) {
-      shutdownAndRemoveInterpreterGroup(interpreterGroupKey);
+    for (InterpreterGroup interpreterGroup : interpreterGroupRef.values()) {
+      interpreterGroup.shutdown();
     }
   }
 

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java

@@ -514,7 +514,8 @@ public class InterpreterSettingManager {
         }
       }
     } catch (NullPointerException e) {
-      logger.warn("Couldn't get interpreter editor setting");
+      // Use `debug` level because this log occurs frequently
+      logger.debug("Couldn't get interpreter editor setting");
     }
     return editor;
   }
@@ -799,19 +800,11 @@ public class InterpreterSettingManager {
 
   public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
       String noteId) {
-    InterpreterOption option = interpreterSetting.getOption();
-    if (option.isProcess()) {
-      interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
-    } else if (option.isSession()) {
-      InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
-      String key = getInterpreterSessionKey(user, noteId, interpreterSetting);
-      interpreterGroup.close(key);
-      synchronized (interpreterGroup) {
-        interpreterGroup.remove(key);
-        interpreterGroup.notifyAll(); // notify createInterpreterForNote()
-      }
-      logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(),
-          noteId);
+    //TODO(jl): This is only for hotfix. You should fix it as a beautiful way
+    InterpreterOption interpreterOption = interpreterSetting.getOption();
+    if (!(InterpreterOption.SHARED.equals(interpreterOption.perNote)
+        && InterpreterOption.SHARED.equals(interpreterOption.perUser))) {
+      interpreterSetting.closeAndRemoveInterpreterGroup(noteId, "");
     }
   }
 
@@ -934,25 +927,8 @@ public class InterpreterSettingManager {
   public void restart(String settingId, String noteId, String user) {
     InterpreterSetting intpSetting = interpreterSettings.get(settingId);
     Preconditions.checkNotNull(intpSetting);
-
-    // restart interpreter setting in note page
-    if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
-      intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
-      return;
-    } else {
-      // restart interpreter setting in interpreter setting page
-      restart(settingId, user);
-    }
-
-  }
-
-  private boolean noteIdIsExist(String noteId) {
-    return noteId == null ? false : true;
-  }
-
-  public void restart(String id, String user) {
     synchronized (interpreterSettings) {
-      InterpreterSetting intpSetting = interpreterSettings.get(id);
+      intpSetting = interpreterSettings.get(settingId);
       // Check if dependency in specified path is changed
       // If it did, overwrite old dependency jar with new one
       if (intpSetting != null) {
@@ -964,17 +940,17 @@ public class InterpreterSettingManager {
         if (user.equals("anonymous")) {
           intpSetting.closeAndRemoveAllInterpreterGroups();
         } else {
-          intpSetting.closeAndRemoveInterpreterGroupByUser(user);
+          intpSetting.closeAndRemoveInterpreterGroup(noteId, user);
         }
 
       } else {
-        throw new InterpreterException("Interpreter setting id " + id + " not found");
+        throw new InterpreterException("Interpreter setting id " + settingId + " not found");
       }
     }
   }
 
   public void restart(String id) {
-    restart(id, "anonymous");
+    restart(id, "", "anonymous");
   }
 
   private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
@@ -1075,6 +1051,10 @@ public class InterpreterSettingManager {
     }
   }
 
+  public void close(InterpreterSetting interpreterSetting) {
+    interpreterSetting.closeAndRemoveAllInterpreterGroups();
+  }
+
   public void close() {
     List<Thread> closeThreads = new LinkedList<>();
     synchronized (interpreterSettings) {

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java

@@ -322,6 +322,13 @@ public class Notebook implements NoteEventListener {
     }
   }
 
+  public void moveNoteToTrash(String noteId) {
+    for (InterpreterSetting interpreterSetting : interpreterSettingManager
+        .getInterpreterSettings(noteId)) {
+      interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId);
+    }
+  }
+
   public void removeNote(String id, AuthenticationInfo subject) {
     Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
 

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java

@@ -95,7 +95,7 @@ public class GitNotebookRepo extends VFSNotebookRepo {
         LOG.debug("No changes found {}", pattern);
       }
     } catch (GitAPIException e) {
-      LOG.error("Failed to add+comit {} to Git", pattern, e);
+      LOG.error("Failed to add+commit {} to Git", pattern, e);
     }
     return revision;
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java

@@ -75,7 +75,6 @@ public class NotebookRepoSync implements NotebookRepo {
     }
 
     for (int i = 0; i < Math.min(storageClassNames.length, getMaxRepoNum()); i++) {
-      @SuppressWarnings("static-access")
       Class<?> notebookStorageClass;
       try {
         notebookStorageClass = getClass().forName(storageClassNames[i].trim());

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java

@@ -22,8 +22,6 @@ import java.net.URISyntaxException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -32,8 +30,11 @@ import org.apache.zeppelin.notebook.NoteInfo;
 import org.apache.zeppelin.notebook.repo.NotebookRepo;
 import org.apache.zeppelin.notebook.repo.NotebookRepoSettingsInfo;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.model.Instance;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserTokenContainer;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserSessionContainer;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.Client;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,27 +56,27 @@ public class ZeppelinHubRepo implements NotebookRepo {
   public static final String TOKEN_HEADER = "X-Zeppelin-Token";
   private static final Gson GSON = new Gson();
   private static final Note EMPTY_NOTE = new Note();
-  //private final Client websocketClient;
+  private final Client websocketClient;
+  private final UserTokenContainer tokenManager;
 
   private String token;
   private ZeppelinhubRestApiHandler restApiClient;
   
   private final ZeppelinConfiguration conf;
-
-  // In order to avoid too many call to ZeppelinHub backend, we save a map of user -> session.
-  private ConcurrentMap<String, String> usersToken = new ConcurrentHashMap<String, String>();
   
   public ZeppelinHubRepo(ZeppelinConfiguration conf) {
     this.conf = conf;
     String zeppelinHubUrl = getZeppelinHubUrl(conf);
     LOG.info("Initializing ZeppelinHub integration module");
-    token = conf.getString("ZEPPELINHUB_API_TOKEN", ZEPPELIN_CONF_PROP_NAME_TOKEN, "");
-    restApiClient = ZeppelinhubRestApiHandler.newInstance(zeppelinHubUrl, token);
 
-    // TODO(xxx): refactor this in the next itaration
-    //websocketClient = Client.initialize(getZeppelinWebsocketUri(conf),
-    //    getZeppelinhubWebsocketUri(conf), token, conf);
-    //websocketClient.start();
+    token = conf.getString("ZEPPELINHUB_API_TOKEN", ZEPPELIN_CONF_PROP_NAME_TOKEN, "");
+    restApiClient = ZeppelinhubRestApiHandler.newInstance(zeppelinHubUrl);
+    //TODO(khalid): check which realm for authentication, pass to token manager
+    tokenManager = UserTokenContainer.init(restApiClient, token);
+
+    websocketClient = Client.initialize(getZeppelinWebsocketUri(conf),
+        getZeppelinhubWebsocketUri(conf), token, conf);
+    websocketClient.start();
   }
 
   private String getZeppelinHubWsUri(URI api) throws URISyntaxException {
@@ -155,58 +156,6 @@ public class ZeppelinHubRepo implements NotebookRepo {
     }
     return zeppelinhubUrl;
   }
-  
-  /**
-   * Get list of user instances from Zeppelinhub.
-   * This will avoid and remove the needs of setting up token in zeppelin-env.sh.
-   */
-  private List<Instance> getUserInstances(String ticket) throws IOException {
-    if (StringUtils.isBlank(ticket)) {
-      return Collections.emptyList();
-    }
-    return restApiClient.getInstances(ticket);
-  }
-
-  /**
-   * Get user default instance.
-   * From now, it will be from the first instance from the list,
-   * But later we can think about marking a default one and return it instead :)
-   */
-  private String getDefaultZeppelinInstanceToken(String ticket) throws IOException {
-    List<Instance> instances = getUserInstances(ticket);
-    if (instances.isEmpty()) {
-      return StringUtils.EMPTY;
-    }
-
-    String token = instances.get(0).token;
-    LOG.debug("The following instance has been assigned {} with token {}", instances.get(0).name,
-        token);
-    return token;
-  }
-
-  /**
-   * For a given user logged in is zeppelin (via zeppelinhub notebook repo), get default token.
-   *
-   */
-  private String getUserToken(String principal) {
-    // Case of user use token instead of authentication.
-    if (!StringUtils.isBlank(token)) {
-      return token;
-    }
-
-    String token = usersToken.get(principal);
-    if (StringUtils.isBlank(token)) {
-      String ticket = UserSessionContainer.instance.getSession(principal);
-      try {
-        token = getDefaultZeppelinInstanceToken(ticket);
-        usersToken.putIfAbsent(principal, token);
-      } catch (IOException e) {
-        LOG.error("Cannot get user token", e);
-        token = StringUtils.EMPTY;
-      }
-    }
-    return token;
-  }
 
   private boolean isSubjectValid(AuthenticationInfo subject) {
     if (subject == null) {
@@ -292,7 +241,6 @@ public class ZeppelinHubRepo implements NotebookRepo {
       return EMPTY_NOTE;
     }
     String endpoint = Joiner.on("/").join(noteId, "checkpoint", revId);
-    
     String token = getUserToken(subject.getUser());
     String response = restApiClient.get(token, endpoint);
 
@@ -320,6 +268,10 @@ public class ZeppelinHubRepo implements NotebookRepo {
     }
     return history;
   }
+  
+  private String getUserToken(String user) {
+    return tokenManager.getUserToken(user);
+  }
 
   @Override
   public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
@@ -335,7 +287,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
     List<Map<String, String>> values = Lists.newLinkedList();
 
     try {
-      instances = getUserInstances(zeppelinHubUserSession);
+      instances = tokenManager.getUserInstances(zeppelinHubUserSession);
     } catch (IOException e) {
       LOG.warn("Couldnt find instances for the session {}, returning empty collection",
           zeppelinHubUserSession);
@@ -368,19 +320,24 @@ public class ZeppelinHubRepo implements NotebookRepo {
     LOG.info("User {} will switch instance", user);
     String ticket = UserSessionContainer.instance.getSession(user);
     List<Instance> instances;
+    String currentToken = StringUtils.EMPTY, targetToken = StringUtils.EMPTY;
     try {
-      instances = getUserInstances(ticket);
+      instances = tokenManager.getUserInstances(ticket);
       if (instances.isEmpty()) {
         return;
       }
-
+      currentToken = tokenManager.getExistingUserToken(user);
       for (Instance instance : instances) {
         if (instance.id == instanceId) {
-          LOG.info("User {} switched to instance {}", user, instances.get(0).name);
-          usersToken.put(user, instance.token);
+          LOG.info("User {} switched to instance {}", user, instance.name);
+          tokenManager.setUserToken(user, instance.token);
+          targetToken = instance.token;
           break;
         }
       }
+      if (!StringUtils.isBlank(currentToken) && !StringUtils.isBlank(targetToken)) {
+        ZeppelinhubUtils.userSwitchTokenRoutine(user, currentToken, targetToken);
+      }
     } catch (IOException e) {
       LOG.error("Cannot switch instance for user {}", user, e);
     }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/model/UserSessionContainer.java

@@ -28,7 +28,7 @@ import org.apache.commons.lang.StringUtils;
 public class UserSessionContainer {
   private static class Entity {
     public final String userSession;
-
+    
     Entity(String userSession) {
       this.userSession = userSession;
     }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/model/UserTokenContainer.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.notebook.repo.zeppelinhub.model;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * User token manager class.
+ *
+ */
+
+public class UserTokenContainer {
+  private static final Logger LOG = LoggerFactory.getLogger(UserTokenContainer.class);
+  private static UserTokenContainer instance = null;
+  private ConcurrentMap<String, String> userTokens = new ConcurrentHashMap<String, String>();
+  private final ZeppelinhubRestApiHandler restApiClient;
+  private String defaultToken;
+
+  public static UserTokenContainer init(ZeppelinhubRestApiHandler restClient, 
+      String defaultToken) {
+    if (instance == null) {
+      instance = new UserTokenContainer(restClient, defaultToken);
+    }
+    return instance;
+  }
+
+  private UserTokenContainer(ZeppelinhubRestApiHandler restClient, String defaultToken) {
+    restApiClient = restClient;
+    this.defaultToken = defaultToken;
+  }
+
+  public static UserTokenContainer getInstance() {
+    return instance;
+  }
+  
+  public void setUserToken(String username, String token) {
+    if (StringUtils.isBlank(username) || StringUtils.isBlank(token)) {
+      LOG.warn("Can't set empty user token");
+      return;
+    }
+    userTokens.put(username, token);
+  }
+  
+  public String getUserToken(String principal) {
+    if (StringUtils.isBlank(principal) || "anonymous".equals(principal)) {
+      if (StringUtils.isBlank(defaultToken)) {
+        return StringUtils.EMPTY;
+      } else {
+        userTokens.putIfAbsent(principal, defaultToken);
+        return defaultToken;
+      }
+    }
+    String token = userTokens.get(principal);
+    if (StringUtils.isBlank(token)) {
+      String ticket = UserSessionContainer.instance.getSession(principal);
+      try {
+        token = getDefaultZeppelinInstanceToken(ticket);
+        if (StringUtils.isBlank(token)) {
+          if (!StringUtils.isBlank(defaultToken)) {
+            token = defaultToken;
+          }
+        } else {
+          userTokens.putIfAbsent(principal, token);
+        }
+      } catch (IOException e) {
+        LOG.error("Cannot get user token", e);
+        token = StringUtils.EMPTY;
+      }
+    }
+    return token;
+  }
+  
+  public String getExistingUserToken(String principal) {
+    if (StringUtils.isBlank(principal) || "anonymous".equals(principal)) {
+      return StringUtils.EMPTY;
+    }
+    String token = userTokens.get(principal);
+    if (token == null) {
+      return StringUtils.EMPTY;
+    }
+    return token;
+  }
+  
+  public String removeUserToken(String username) {
+    return userTokens.remove(username);
+  }
+  
+  /**
+   * Get user default instance.
+   * From now, it will be from the first instance from the list,
+   * But later we can think about marking a default one and return it instead :)
+   */
+  public String getDefaultZeppelinInstanceToken(String ticket) throws IOException {
+    List<Instance> instances = getUserInstances(ticket);
+    if (instances.isEmpty()) {
+      return StringUtils.EMPTY;
+    }
+
+    String token = instances.get(0).token;
+    LOG.debug("The following instance has been assigned {} with token {}", instances.get(0).name,
+        token);
+    return token;
+  }
+  
+  /**
+   * Get list of user instances from Zeppelinhub.
+   * This will avoid and remove the needs of setting up token in zeppelin-env.sh.
+   */
+  public List<Instance> getUserInstances(String ticket) throws IOException {
+    if (StringUtils.isBlank(ticket)) {
+      return Collections.emptyList();
+    }
+    return restApiClient.getInstances(ticket);
+  }
+  
+  public List<String> getAllTokens() {
+    return new ArrayList<String>(userTokens.values());
+  }
+  
+  public Map<String, String> getAllUserTokens() {
+    return new HashMap<String, String>(userTokens);
+  }
+}

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/rest/ZeppelinhubRestApiHandler.java

@@ -58,11 +58,11 @@ public class ZeppelinhubRestApiHandler {
   private final HttpClient client;
   private final String zepelinhubUrl;
 
-  public static ZeppelinhubRestApiHandler newInstance(String zeppelinhubUrl, String token) {
-    return new ZeppelinhubRestApiHandler(zeppelinhubUrl, token);
+  public static ZeppelinhubRestApiHandler newInstance(String zeppelinhubUrl) {
+    return new ZeppelinhubRestApiHandler(zeppelinhubUrl);
   }
 
-  private ZeppelinhubRestApiHandler(String zeppelinhubUrl, String token) {
+  private ZeppelinhubRestApiHandler(String zeppelinhubUrl) {
     this.zepelinhubUrl = zeppelinhubUrl + DEFAULT_API_PATH + "/";
 
     //TODO(khalid):to make proxy conf consistent with Zeppelin confs
@@ -155,6 +155,9 @@ public class ZeppelinhubRestApiHandler {
   }
 
   public String get(String token, String argument) throws IOException {
+    if (StringUtils.isBlank(token)) {
+      return StringUtils.EMPTY;
+    }
     String url = zepelinhubUrl + argument;
     return sendToZeppelinHub(HttpMethod.GET, url, StringUtils.EMPTY, token, true);
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/Client.java

@@ -72,8 +72,8 @@ public class Client {
     }
   }
 
-  public void relayToZeppelinHub(String message) {
-    zeppelinhubClient.send(message);
+  public void relayToZeppelinHub(String message, String token) {
+    zeppelinhubClient.send(message, token);
   }
 
   public void relayToZeppelin(Message message, String noteId) {

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClient.java

@@ -18,8 +18,11 @@ package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.Timer;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
@@ -27,6 +30,8 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.NotebookAuthorization;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserTokenContainer;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.security.Authentication;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.WatcherWebsocket;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.ZeppelinWebsocket;
@@ -53,7 +58,6 @@ import com.google.gson.JsonSyntaxException;
 public class ZeppelinClient {
   private static final Logger LOG = LoggerFactory.getLogger(ZeppelinClient.class);
   private final URI zeppelinWebsocketUrl;
-  private final String zeppelinhubToken;
   private final WebSocketClient wsClient;
   private static Gson gson;
   // Keep track of current open connection per notebook.
@@ -64,6 +68,21 @@ public class ZeppelinClient {
   private SchedulerService schedulerService;
   private Authentication authModule;
   private static final int MIN = 60;
+  private static final String ORIGIN = "Origin";
+
+  private static final Set<String> actionable = new  HashSet<String>(Arrays.asList(
+      // running events
+      "ANGULAR_OBJECT_UPDATE",
+      "PROGRESS",
+      "NOTE",
+      "PARAGRAPH",
+      "PARAGRAPH_UPDATE_OUTPUT",
+      "PARAGRAPH_APPEND_OUTPUT",
+      "PARAGRAPH_CLEAR_OUTPUT",
+      "PARAGRAPH_REMOVE",
+      // run or stop events
+      "RUN_PARAGRAPH",
+      "CANCEL_PARAGRAPH"));
 
   public static ZeppelinClient initialize(String zeppelinUrl, String token, 
       ZeppelinConfiguration conf) {
@@ -79,7 +98,6 @@ public class ZeppelinClient {
 
   private ZeppelinClient(String zeppelinUrl, String token, ZeppelinConfiguration conf) {
     zeppelinWebsocketUrl = URI.create(zeppelinUrl);
-    zeppelinhubToken = token;
     wsClient = createNewWebsocketClient();
     gson = new Gson();
     notesConnection = new ConcurrentHashMap<>();
@@ -121,7 +139,7 @@ public class ZeppelinClient {
       public void run() {
         watcherSession = openWatcherSession();
       }
-    }, 5000);
+    }, 10000);
   }
 
   public void stop() {
@@ -171,6 +189,7 @@ public class ZeppelinClient {
   private Session openWatcherSession() {
     ClientUpgradeRequest request = new ClientUpgradeRequest();
     request.setHeader(WatcherSecurityKey.HTTP_HEADER, WatcherSecurityKey.getKey());
+    request.setHeader(ORIGIN, "*");
     WatcherWebsocket socket = WatcherWebsocket.createInstace();
     Future<Session> future = null;
     Session session = null;
@@ -185,7 +204,7 @@ public class ZeppelinClient {
   }
 
   public void send(Message msg, String noteId) {
-    Session noteSession = getZeppelinConnection(noteId);
+    Session noteSession = getZeppelinConnection(noteId, msg.principal, msg.ticket);
     if (!isSessionOpen(noteSession)) {
       LOG.error("Cannot open websocket connection to Zeppelin note {}", noteId);
       return;
@@ -193,12 +212,12 @@ public class ZeppelinClient {
     noteSession.getRemote().sendStringByFuture(serialize(msg));
   }
   
-  public Session getZeppelinConnection(String noteId) {
+  public Session getZeppelinConnection(String noteId, String principal, String ticket) {
     if (StringUtils.isBlank(noteId)) {
       LOG.warn("Cannot get Websocket session with blanck noteId");
       return null;
     }
-    return getNoteSession(noteId);
+    return getNoteSession(noteId, principal, ticket);
   }
   
 /*
@@ -211,19 +230,20 @@ public class ZeppelinClient {
   }
   */
 
-  private Session getNoteSession(String noteId) {
+  private Session getNoteSession(String noteId, String principal, String ticket) {
     LOG.info("Getting Note websocket connection for note {}", noteId);
     Session session = notesConnection.get(noteId);
     if (!isSessionOpen(session)) {
       LOG.info("No open connection for note {}, opening one", noteId);
       notesConnection.remove(noteId);
-      session = openNoteSession(noteId);
+      session = openNoteSession(noteId, principal, ticket);
     }
     return session;
   }
   
-  private Session openNoteSession(String noteId) {
+  private Session openNoteSession(String noteId, String principal, String ticket) {
     ClientUpgradeRequest request = new ClientUpgradeRequest();
+    request.setHeader(ORIGIN, "*");
     ZeppelinWebsocket socket = new ZeppelinWebsocket(noteId);
     Future<Session> future = null;
     Session session = null;
@@ -239,7 +259,7 @@ public class ZeppelinClient {
       session.close();
       session = notesConnection.get(noteId);
     } else {
-      String getNote = serialize(zeppelinGetNoteMsg(noteId));
+      String getNote = serialize(zeppelinGetNoteMsg(noteId, principal, ticket));
       session.getRemote().sendStringByFuture(getNote);
       notesConnection.put(noteId, session);
     }
@@ -250,31 +270,71 @@ public class ZeppelinClient {
     return (session != null) && (session.isOpen());
   }
 
-  private Message zeppelinGetNoteMsg(String noteId) {
+  private Message zeppelinGetNoteMsg(String noteId, String principal, String ticket) {
     Message getNoteMsg = new Message(Message.OP.GET_NOTE);
     HashMap<String, Object> data = new HashMap<String, Object>();
     data.put("id", noteId);
     getNoteMsg.data = data;
+    getNoteMsg.principal = principal;
+    getNoteMsg.ticket = ticket;
     return getNoteMsg;
   }
 
   public void handleMsgFromZeppelin(String message, String noteId) {
     Map<String, String> meta = new HashMap<>();
-    meta.put("token", zeppelinhubToken);
+    //TODO(khalid): don't use zeppelinhubToken in this class, decouple
     meta.put("noteId", noteId);
     Message zeppelinMsg = deserialize(message);
     if (zeppelinMsg == null) {
       return;
     }
-    ZeppelinhubMessage hubMsg = ZeppelinhubMessage.newMessage(zeppelinMsg, meta);
+    String token;
+    if (!isActionable(zeppelinMsg.op)) {
+      return;
+    }
+    
+    token = UserTokenContainer.getInstance().getUserToken(zeppelinMsg.principal);
     Client client = Client.getInstance();
     if (client == null) {
       LOG.warn("Client isn't initialized yet");
       return;
     }
-    client.relayToZeppelinHub(hubMsg.serialize());
+    ZeppelinhubMessage hubMsg = ZeppelinhubMessage.newMessage(zeppelinMsg, meta);
+    if (StringUtils.isEmpty(token)) {
+      relayToAllZeppelinHub(hubMsg, noteId);
+    } else {
+      client.relayToZeppelinHub(hubMsg.serialize(), token);
+    }
+
   }
 
+  private void relayToAllZeppelinHub(ZeppelinhubMessage hubMsg, String noteId) {
+    if (StringUtils.isBlank(noteId)) {
+      return;
+    }
+    NotebookAuthorization noteAuth = NotebookAuthorization.getInstance();
+    Map<String, String> userTokens = UserTokenContainer.getInstance().getAllUserTokens();
+    Client client = Client.getInstance();
+    Set<String> userAndRoles;
+    String token;
+    for (String user: userTokens.keySet()) {
+      userAndRoles = noteAuth.getRoles(user);
+      userAndRoles.add(user);
+      if (noteAuth.isReader(noteId, userAndRoles)) {
+        token = userTokens.get(user);
+        hubMsg.meta.put("token", token);
+        client.relayToZeppelinHub(hubMsg.serialize(), token);
+      }
+    }
+  }
+
+  private boolean isActionable(OP action) {
+    if (action == null) {
+      return false;
+    }
+    return actionable.contains(action.name());
+  }
+  
   public void removeNoteConnection(String noteId) {
     if (StringUtils.isBlank(noteId)) {
       LOG.error("Cannot remove session for empty noteId");
@@ -307,7 +367,7 @@ public class ZeppelinClient {
 
   public void ping() {
     if (watcherSession == null) {
-      LOG.info("Cannot send PING event, no watcher found");
+      LOG.debug("Cannot send PING event, no watcher found");
       return;
     }
     watcherSession.getRemote().sendStringByFuture(serialize(new Message(OP.PING)));

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClient.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.net.HttpCookie;
 import java.net.URI;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
@@ -36,6 +37,7 @@ import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.session.Zeppelinh
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
 import org.apache.zeppelin.notebook.socket.Message;
 import org.apache.zeppelin.notebook.socket.Message.OP;
+import org.apache.zeppelin.ticket.TicketContainer;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.websocket.api.Session;
 import org.eclipse.jetty.websocket.client.ClientUpgradeRequest;
@@ -58,7 +60,6 @@ public class ZeppelinhubClient {
 
   private final WebSocketClient client;
   private final URI zeppelinhubWebsocketUrl;
-  private final ClientUpgradeRequest conectionRequest;
   private final String zeppelinhubToken;
 
   private static final long CONNECTION_IDLE_TIME = TimeUnit.SECONDS.toMillis(30);
@@ -66,7 +67,8 @@ public class ZeppelinhubClient {
   private static Gson gson;
   
   private SchedulerService schedulerService;
-  private ZeppelinhubSession zeppelinhubSession;
+  private Map<String, ZeppelinhubSession> sessionMap = 
+      new ConcurrentHashMap<String, ZeppelinhubSession>();
 
   public static ZeppelinhubClient initialize(String zeppelinhubUrl, String token) {
     if (instance == null) {
@@ -82,7 +84,6 @@ public class ZeppelinhubClient {
   private ZeppelinhubClient(String url, String token) {
     zeppelinhubWebsocketUrl = URI.create(url);
     client = createNewWebsocketClient();
-    conectionRequest = setConnectionrequest(token);
     zeppelinhubToken = token;
     schedulerService = SchedulerService.create(10);
     gson = new Gson();
@@ -92,17 +93,19 @@ public class ZeppelinhubClient {
   public void start() {
     try {
       client.start();
-      zeppelinhubSession = connect();
       addRoutines();
     } catch (Exception e) {
       LOG.error("Cannot connect to zeppelinhub via websocket", e);
     }
   }
   
+  public void initUser(String token) {
+    
+  }
+
   public void stop() {
     LOG.info("Stopping Zeppelinhub websocket client");
     try {
-      zeppelinhubSession.close();
       schedulerService.close();
       client.stop();
     } catch (Exception e) {
@@ -110,14 +113,19 @@ public class ZeppelinhubClient {
     }
   }
 
+  public void stopUser(String token) {
+    removeSession(token);
+  }
+
   public String getToken() {
     return this.zeppelinhubToken;
   }
   
-  public void send(String msg) {
-    if (!isConnectedToZeppelinhub()) {
+  public void send(String msg, String token) {
+    ZeppelinhubSession zeppelinhubSession = getSession(token);
+    if (!isConnectedToZeppelinhub(zeppelinhubSession)) {
       LOG.info("Zeppelinhub connection is not open, opening it");
-      zeppelinhubSession = connect();
+      zeppelinhubSession = connect(token);
       if (zeppelinhubSession == ZeppelinhubSession.EMPTY) {
         LOG.warn("While connecting to ZeppelinHub received empty session, cannot send the message");
         return;
@@ -126,25 +134,48 @@ public class ZeppelinhubClient {
     zeppelinhubSession.sendByFuture(msg);
   }
   
-  private boolean isConnectedToZeppelinhub() {
+  private boolean isConnectedToZeppelinhub(ZeppelinhubSession zeppelinhubSession) {
     return (zeppelinhubSession != null && zeppelinhubSession.isSessionOpen());
   }
 
-  private ZeppelinhubSession connect() {
-    ZeppelinhubSession zeppelinSession;
-    try {
-      ZeppelinhubWebsocket ws = ZeppelinhubWebsocket.newInstance(zeppelinhubToken);
-      Future<Session> future = client.connect(ws, zeppelinhubWebsocketUrl, conectionRequest);
-      Session session = future.get();
-      zeppelinSession = ZeppelinhubSession.createInstance(session, zeppelinhubToken);
-    } catch (IOException | InterruptedException | ExecutionException e) {
-      LOG.info("Couldnt connect to zeppelinhub - {}", e.toString());
-      zeppelinSession = ZeppelinhubSession.EMPTY;
+  private ZeppelinhubSession connect(String token) {
+    if (StringUtils.isBlank(token)) {
+      LOG.debug("Can't connect with empty token");
+      return ZeppelinhubSession.EMPTY;
     }
-    return zeppelinSession;
+    ZeppelinhubSession zeppelinhubSession;
+    try {
+      ZeppelinhubWebsocket ws = ZeppelinhubWebsocket.newInstance(token);
+      ClientUpgradeRequest request = getConnectionRequest(token);
+      Future<Session> future = client.connect(ws, zeppelinhubWebsocketUrl, request);
+      Session session = future.get();
+      zeppelinhubSession = ZeppelinhubSession.createInstance(session, token);
+      setSession(token, zeppelinhubSession);
+    } catch (IOException | InterruptedException | ExecutionException e) {
+      LOG.info("Couldnt connect to zeppelinhub", e);
+      zeppelinhubSession = ZeppelinhubSession.EMPTY;
+    }
+    return zeppelinhubSession;
   }
-  
-  private ClientUpgradeRequest setConnectionrequest(String token) {
+
+  private void setSession(String token, ZeppelinhubSession session) {
+    sessionMap.put(token, session);
+  }
+
+  private ZeppelinhubSession getSession(String token) {
+    return sessionMap.get(token);
+  }
+
+  public void removeSession(String token) {
+    ZeppelinhubSession zeppelinhubSession = getSession(token);
+    if (zeppelinhubSession == null) {
+      return;
+    }
+    zeppelinhubSession.close();
+    sessionMap.remove(token);
+  }
+
+  private ClientUpgradeRequest getConnectionRequest(String token) {
     ClientUpgradeRequest request = new ClientUpgradeRequest();
     request.setCookies(Lists.newArrayList(new HttpCookie(ZeppelinHubRepo.TOKEN_HEADER, token)));
     return request;
@@ -193,7 +224,7 @@ public class ZeppelinhubClient {
           runAllParagraph(hubMsg.meta.get("noteId"), msg);
           break;
         default:
-          LOG.warn("Received {} from ZeppelinHub, not handled", op);
+          LOG.debug("Received {} from ZeppelinHub, not handled", op);
           break;
     }
   }
@@ -206,6 +237,8 @@ public class ZeppelinhubClient {
       return;
     }
     zeppelinMsg.data = (Map<String, Object>) hubMsg.data;
+    zeppelinMsg.principal = hubMsg.meta.get("owner");
+    zeppelinMsg.ticket = TicketContainer.instance.getTicket(zeppelinMsg.principal);
     Client client = Client.getInstance();
     if (client == null) {
       LOG.warn("Base client isn't initialized, returning");
@@ -230,6 +263,7 @@ public class ZeppelinhubClient {
       Message zeppelinMsg = new Message(OP.RUN_PARAGRAPH);
 
       JSONArray paragraphs = data.getJSONArray("data");
+      String principal = data.getJSONObject("meta").getString("owner");
       for (int i = 0; i < paragraphs.length(); i++) {
         if (!(paragraphs.get(i) instanceof JSONObject)) {
           LOG.warn("Wrong \"paragraph\" format for RUN_NOTEBOOK");
@@ -237,6 +271,8 @@ public class ZeppelinhubClient {
         }
         zeppelinMsg.data = gson.fromJson(paragraphs.getString(i), 
             new TypeToken<Map<String, Object>>(){}.getType());
+        zeppelinMsg.principal = principal;
+        zeppelinMsg.ticket = TicketContainer.instance.getTicket(principal);
         client.relayToZeppelin(zeppelinMsg, noteId);
         LOG.info("\nSending RUN_PARAGRAPH message to Zeppelin ");
       }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/listener/WatcherWebsocket.java

@@ -21,6 +21,7 @@ import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinClient;
 import org.apache.zeppelin.notebook.socket.Message;
 import org.apache.zeppelin.notebook.socket.Message.OP;
 import org.apache.zeppelin.notebook.socket.WatcherMessage;
+import org.apache.zeppelin.ticket.TicketContainer;
 import org.eclipse.jetty.websocket.api.Session;
 import org.eclipse.jetty.websocket.api.WebSocketListener;
 import org.slf4j.Logger;
@@ -35,6 +36,7 @@ import com.google.gson.Gson;
 public class WatcherWebsocket implements WebSocketListener {
   private static final Logger LOG = LoggerFactory.getLogger(ZeppelinWebsocket.class);
   private static final Gson GSON = new Gson();
+  private static final String watcherPrincipal = "watcher";
   public Session connection;
   
   public static WatcherWebsocket createInstace() {
@@ -54,7 +56,10 @@ public class WatcherWebsocket implements WebSocketListener {
   public void onWebSocketConnect(Session session) {
     LOG.info("WatcherWebsocket connection opened");
     this.connection = session;
-    session.getRemote().sendStringByFuture(GSON.toJson(new Message(OP.WATCHER)));
+    Message watcherMsg = new Message(OP.WATCHER);
+    watcherMsg.principal = watcherPrincipal;
+    watcherMsg.ticket = TicketContainer.instance.getTicket(watcherPrincipal);
+    session.getRemote().sendStringByFuture(GSON.toJson(watcherMsg));
   }
 
   @Override

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/scheduler/ZeppelinHubHeartbeat.java

@@ -16,6 +16,7 @@
  */
 package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler;
 
+import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserTokenContainer;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
 import org.slf4j.Logger;
@@ -39,7 +40,9 @@ public class ZeppelinHubHeartbeat implements Runnable {
   
   @Override
   public void run() {
-    LOG.debug("Sending PING to zeppelinhub");
-    client.send(ZeppelinhubUtils.pingMessage(client.getToken()));
+    LOG.debug("Sending PING to zeppelinhub token");
+    for (String token: UserTokenContainer.getInstance().getAllTokens()) {
+      client.send(ZeppelinhubUtils.pingMessage(token), token);
+    }
   }  
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/utils/ZeppelinhubUtils.java

@@ -19,6 +19,8 @@ package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils;
 import java.util.HashMap;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserTokenContainer;
+import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinHubOp;
 import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinhubMessage;
 import org.apache.zeppelin.notebook.socket.Message;
@@ -95,4 +97,32 @@ public class ZeppelinhubUtils {
   public static boolean isZeppelinOp(String text) {
     return (toZeppelinOp(text) != null); 
   }
+  
+  public static void userLoginRoutine(String username) {
+    LOG.debug("Executing user login routine");
+    String token = UserTokenContainer.getInstance().getUserToken(username);
+    UserTokenContainer.getInstance().setUserToken(username, token);
+    String msg = ZeppelinhubUtils.liveMessage(token);
+    ZeppelinhubClient.getInstance()
+        .send(msg, token);
+  }
+  
+  public static void userLogoutRoutine(String username) {
+    LOG.debug("Executing user logout routine");
+    String token = UserTokenContainer.getInstance().removeUserToken(username);
+    String msg = ZeppelinhubUtils.deadMessage(token);
+    ZeppelinhubClient.getInstance()
+        .send(msg, token);
+    ZeppelinhubClient.getInstance().removeSession(token);
+  }
+  
+  public static void userSwitchTokenRoutine(String username, String originToken,
+      String targetToken) {
+    String offMsg = ZeppelinhubUtils.deadMessage(originToken);
+    ZeppelinhubClient.getInstance().send(offMsg, originToken);
+    ZeppelinhubClient.getInstance().removeSession(originToken);
+    
+    String onMsg = ZeppelinhubUtils.liveMessage(targetToken);
+    ZeppelinhubClient.getInstance().send(onMsg, targetToken);
+  }
 }

zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java

@@ -30,10 +30,13 @@ import java.net.URL;
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
 import static org.junit.Assert.*;
 
 public class HeliumBundleFactoryTest {
   private File tmpDir;
+  private ZeppelinConfiguration conf;
   private HeliumBundleFactory hbf;
 
   @Before
@@ -46,7 +49,10 @@ public class HeliumBundleFactoryTest {
     String resDir = new File(res.getFile()).getParent();
     File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/");
 
-    hbf = new HeliumBundleFactory(tmpDir,
+    conf = new ZeppelinConfiguration();
+
+    hbf = new HeliumBundleFactory(conf,
+        tmpDir,
         new File(moduleDir, "tabledata"),
         new File(moduleDir, "visualization"),
         new File(moduleDir, "spell"));
@@ -59,6 +65,13 @@ public class HeliumBundleFactoryTest {
 
   @Test
   public void testInstallNpm() throws InstallationException {
+    assertFalse(new File(tmpDir,
+        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
+    assertFalse(new File(tmpDir,
+        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
+
+    hbf.installNodeAndNpm();
+
     assertTrue(new File(tmpDir,
         HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
     assertTrue(new File(tmpDir,

zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java

@@ -224,7 +224,7 @@ public class InterpreterFactoryTest {
     LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup.get("user2").get(0);
     interpreter2.open();
 
-    mock1Setting.closeAndRemoveInterpreterGroupByUser("user1");
+    mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
     assertFalse(interpreter1.isOpen());
     assertTrue(interpreter2.isOpen());
   }
@@ -270,7 +270,7 @@ public class InterpreterFactoryTest {
     LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup2.get("shared_session").get(0);
     interpreter2.open();
 
-    mock1Setting.closeAndRemoveInterpreterGroupByUser("user1");
+    mock1Setting.closeAndRemoveInterpreterGroup("note1", "user1");
     assertFalse(interpreter1.isOpen());
     assertTrue(interpreter2.isOpen());
   }
@@ -473,13 +473,13 @@ public class InterpreterFactoryTest {
     InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
     String testInterpreterRunner = "relativePath.sh";
     when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
-    Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), "settingId", "userName", false, mockInterpreterRunner);
+    Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
     String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
     assertNotEquals(interpreterRunner, testInterpreterRunner);
 
     testInterpreterRunner = "/AbsolutePath.sh";
     when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
-    i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), "settingId", "userName", false, mockInterpreterRunner);
+    i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
     interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
     assertEquals(interpreterRunner, testInterpreterRunner);
   }