Merge branch 'master' into doc-formatting-fixes

bin/common.cmd

@@ -71,6 +71,14 @@ if not defined ZEPPELIN_JAVA_OPTS (
     set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
 )
 
+if defined ZEPPELIN_JMX_ENABLE (
+  if not defined ZEPPELIN_JMX_PORT (
+    set ZEPPELIN_JMX_PORT="9996"
+  }
+  set JMX_JAVA_OPTS=" -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+  set ZEPPELIN_JAVA_OPTS=%JMX_JAVA_OPTS% %ZEPPELIN_JAVA_OPTS
+)
+
 if not defined JAVA_OPTS (
     set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
 ) else (

bin/common.sh

@@ -121,6 +121,18 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
 JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
 export JAVA_OPTS
 
+if [[ x"${ZEPPELIN_JMX_ENABLE}" == x"true" ]]; then
+  if [[ -z "${ZEPPELIN_JMX_PORT}" ]]; then
+    ZEPPELIN_JMX_PORT="9996"
+  fi
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT}"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.authenticate=false"
+  JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.ssl=false"
+  JAVA_OPTS="${JMX_JAVA_OPTS} ${JAVA_OPTS}"
+fi
+export JAVA_OPTS
+
 JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
 if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
     JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"

bin/interpreter.sh

@@ -24,7 +24,7 @@ function usage() {
     echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
 }
 
-while getopts "hc:p:r:d:l:v:u:g:" o; do
+while getopts "hc:p:r:i:d:l:v:u:g:" o; do
     case ${o} in
         h)
             usage
@@ -42,6 +42,9 @@ while getopts "hc:p:r:d:l:v:u:g:" o; do
         r)
             INTP_PORT=${OPTARG} # This will be used for interpreter process port
             ;;
+        i)
+            INTP_GROUP_ID=${OPTARG} # This will be used for interpreter group id
+            ;;
         l)
             LOCAL_INTERPRETER_REPO=${OPTARG}
             ;;
@@ -218,9 +221,9 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
 fi
 
 if [[ -n "${SPARK_SUBMIT}" ]]; then
-    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
 else
-    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
+    INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
 fi
 
 

cassandra/src/main/resources/interpreter-setting.json

@@ -224,7 +224,7 @@
       "cassandra.ssl.enabled": {
         "envName": null,
         "propertyName": "cassandra.ssl.enabled",
-        "defaultValue": "false",
+        "defaultValue": false,
         "description": "Cassandra SSL",
         "type": "checkbox"
       },

conf/zeppelin-env.cmd.template

@@ -22,6 +22,8 @@ REM set ZEPPELIN_JAVA_OPTS      		REM Additional jvm options. for example, set Z
 REM set ZEPPELIN_MEM            		REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
 REM set ZEPPELIN_INTP_MEM       		REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxPermSize=512m
 REM set ZEPPELIN_INTP_JAVA_OPTS 		REM zeppelin interpreter process jvm options.
+REM set ZEPPELIN_JMX_ENABLE     		REM Enable JMX feature by defining it like "true"
+REM set ZEPPELIN_JMX_PORT       		REM Port number which JMX uses. Default: "9996"
 
 REM set ZEPPELIN_LOG_DIR        		REM Where log files are stored.  PWD by default.
 REM set ZEPPELIN_PID_DIR        		REM The pid files are stored. /tmp by default.

conf/zeppelin-env.sh.template

@@ -23,6 +23,8 @@
 # export ZEPPELIN_INTP_MEM       		# zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
 # export ZEPPELIN_INTP_JAVA_OPTS 		# zeppelin interpreter process jvm options.
 # export ZEPPELIN_SSL_PORT       		# ssl port (used when ssl environment variable is set to true)
+# export ZEPPELIN_JMX_ENABLE    		# Enable JMX feature by defining "true"
+# export ZEPPELIN_JMX_PORT      		# Port number which JMX uses. Default: "9996"
 
 # export ZEPPELIN_LOG_DIR        		# Where log files are stored.  PWD by default.
 # export ZEPPELIN_PID_DIR        		# The pid files are stored. ${ZEPPELIN_HOME}/run by default.

conf/zeppelin-site.xml.template

@@ -67,6 +67,12 @@
   <description>hide homescreen notebook from list when this value set to true</description>
 </property>
 
+<property>
+  <name>zeppelin.notebook.collaborative.mode.enable</name>
+  <value>true</value>
+  <description>Enable collaborative mode</description>
+</property>
+
 <!-- Google Cloud Storage notebook storage -->
 <!--
 <property>

docs/interpreter/spark.md

@@ -205,8 +205,7 @@ You can either specify them in `zeppelin-env.sh`, or in interpreter setting page
 in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance.
 
 ### 4. New Version of SparkInterpreter
-There's one new version of SparkInterpreter starting with better spark support and code completion from Zeppelin 0.8.0, by default we still use the old version of SparkInterpreter.
-If you want to use the new one, you can configure `zeppelin.spark.useNew` as `true` in its interpreter setting.
+There's one new version of SparkInterpreter with better spark support and code completion starting from Zeppelin 0.8.0. We enable it by default, but user can still use the old version of SparkInterpreter by setting `zeppelin.spark.useNew` as `false` in its interpreter setting.
 
 ## SparkContext, SQLContext, SparkSession, ZeppelinContext
 SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.

docs/setup/operation/configuration.md

@@ -101,6 +101,12 @@ If both are defined, then the **environment variables** will take priority.
     <td>/</td>
     <td>Context path of the web application</td>
   </tr>
+  <tr>
+    <td><h6 class="properties">ZEPPELIN_NOTEBOOK_COLLABORATIVE_MODE_ENABLE</h6></td>
+    <td><h6 class="properties">zeppelin.notebook.collaborative.mode.enable</h6></td>
+    <td>true</td>
+    <td>Enable basic opportunity for collaborative editing. Does not change the logic of operation if the note is used by one person.</td>
+  </tr>
   <tr>
     <td><h6 class="properties">ZEPPELIN_SSL</h6></td>
     <td><h6 class="properties">zeppelin.ssl</h6></td>

docs/setup/security/shiro_authentication.md

@@ -104,6 +104,9 @@ To learn more about Apache Shiro Realm, please check [this documentation](http:/
 
 We also provide community custom Realms.
 
+**Note**: When using any of the below realms the default 
+      password-based (IniRealm) authentication needs to be disabled.
+
 ### Active Directory
 
 ```
@@ -182,6 +185,17 @@ securityManager.sessionManager = $sessionManager
 securityManager.realms = $ldapRealm
 ```
 
+Also instead of specifying systemPassword in clear text in `shiro.ini` administrator can choose to specify the same in "hadoop credential". 
+Create a keystore file using the hadoop credential command line:
+``` 
+hadoop credential create ldapRealm.systemPassword -provider jceks://file/user/zeppelin/conf/zeppelin.jceks
+```
+
+Add the following line in the `shiro.ini` file:
+``` 
+ldapRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks
+```
+
 ### PAM
 [PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication
 moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can
@@ -274,15 +288,16 @@ By default, Shiro will allow access to a URL if only user is part of "**all the
 /api/interpreter/** = authc, roles[admin, role1]
 ```
 
-If there is a need that user with "**any of the defined roles**" should be allowed, then following Shiro configuration can be used:
+### Apply multiple roles or user in Shiro configuration
+If there is a need that user with "**any of the defined roles or user itself**" should be allowed, then following Shiro configuration can be used:
 
 ```
 [main]
-anyofroles = org.apache.zeppelin.utils.AnyOfRolesAuthorizationFilter
+anyofrolesuser = org.apache.zeppelin.utils.AnyOfRolesUserAuthorizationFilter
 
 [urls]
 
-/api/interpreter/** = authc, anyofroles[admin, role1]
+/api/interpreter/** = authc, anyofrolesuser[admin, user1]
 /api/configurations/** = authc, roles[admin]
 /api/credential/** = authc, roles[admin]
 ```

docs/usage/other_features/cron_scheduler.md

@@ -45,7 +45,7 @@ You can set the cron schedule by filling in this form. Please see [Cron Trigger
 
 You can set the cron executing user by filling in this form and press the enter key.
 
-### auto-restart interpreter on cron execution
+### After execution stop the interpreter
 
 When this checkbox is set to "on", the interpreters which are binded to the notebook are stopped automatically after the cron execution. This feature is useful if you want to release the interpreter resources after the cron execution.
 

elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java

@@ -161,9 +161,11 @@ public class ElasticsearchInterpreterTest {
   }
 
   private InterpreterContext buildContext(String noteAndParagraphId) {
-    final AngularObjectRegistry angularObjReg = new AngularObjectRegistry("elasticsearch", null);
-    return new InterpreterContext(noteAndParagraphId, noteAndParagraphId, null, null, null, null,
-        null, null, null, angularObjReg , null, null, null);
+    return InterpreterContext.builder()
+        .setNoteId(noteAndParagraphId)
+        .setParagraphId(noteAndParagraphId)
+        .setAngularObjectRegistry(new AngularObjectRegistry("elasticsearch", null))
+        .build();
   }
 
   @Theory

flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java

@@ -17,9 +17,9 @@
  */
 package org.apache.zeppelin.flink;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -27,9 +27,8 @@ import org.junit.Test;
 import java.util.Arrays;
 import java.util.Properties;
 
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 
 public class FlinkInterpreterTest {
 
@@ -41,8 +40,7 @@ public class FlinkInterpreterTest {
     Properties p = new Properties();
     flink = new FlinkInterpreter(p);
     flink.open();
-    context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null,
-            null, null, null);
+    context = InterpreterContext.builder().build();
   }
 
   @AfterClass

groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java

@@ -16,26 +16,23 @@
  */
 package org.apache.zeppelin.groovy;
 
-import org.slf4j.Logger;
-
-import java.io.StringWriter;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
 import groovy.lang.Closure;
 import groovy.xml.MarkupBuilder;
-
+import org.apache.thrift.TException;
 import org.apache.zeppelin.annotation.ZeppelinApi;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.RemoteWorksController;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
 
 /**
  * Groovy interpreter for Zeppelin.
@@ -46,6 +43,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
   Properties props;
   InterpreterContext interpreterContext;
   Map<String, Object> bindings;
+  GroovyZeppelinContext z;
 
   public GObject(Logger log, StringWriter out, Properties p, InterpreterContext ctx,
       Map<String, Object> bindings) {
@@ -54,6 +52,8 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
     this.interpreterContext = ctx;
     this.props = p;
     this.bindings = bindings;
+    this.z = new GroovyZeppelinContext(null, 1000);
+    this.z.setInterpreterContext(this.interpreterContext);
   }
 
   public Object getProperty(String key) {
@@ -87,17 +87,17 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
    * returns gui object.
    */
   public GUI getGui() {
-    return interpreterContext.getGui();
+    return z.getGui();
   }
 
   @ZeppelinApi
   public Object input(String name) {
-    return input(name, "");
+    return z.input(name, "");
   }
 
   @ZeppelinApi
   public Object input(String name, Object defaultValue) {
-    return getGui().input(name, defaultValue);
+    return z.input(name, defaultValue);
   }
 
   private ParamOption[] toParamOptions(Map<Object, String> options) {
@@ -111,23 +111,23 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
 
   @ZeppelinApi
   public Object select(String name, Map<Object, String> options) {
-    return select(name, "", options);
+    return z.select(name, "", toParamOptions(options));
   }
 
   @ZeppelinApi
   public Object select(String name, Object defaultValue, Map<Object, String> options) {
-    return getGui().select(name, defaultValue, toParamOptions(options));
+    return z.select(name, defaultValue, toParamOptions(options));
   }
 
   @ZeppelinApi
   public Collection<Object> checkbox(String name, Map<Object, String> options) {
-    return checkbox(name, options.keySet(), options);
+    return z.checkbox(name, new ArrayList<Object>(options.keySet()), toParamOptions(options));
   }
 
   @ZeppelinApi
   public Collection<Object> checkbox(String name, Collection<Object> defaultChecked,
       Map<Object, String> options) {
-    return getGui().checkbox(name, defaultChecked, toParamOptions(options));
+    return z.checkbox(name, new ArrayList<Object>(defaultChecked), toParamOptions(options));
   }
 
 
@@ -236,23 +236,12 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
    * @return value
    */
   public Object angular(String name) {
-    AngularObject ao = getAngularObject(name);
-    if (ao == null) {
-      return null;
-    } else {
-      return ao.get();
-    }
+    return z.angular(name);
   }
 
   @SuppressWarnings("unchecked")
-  public void angularBind(String name, Object o, String noteId) {
-    AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
-
-    if (registry.get(name, noteId, null) == null) {
-      registry.add(name, o, noteId, null);
-    } else {
-      registry.get(name, noteId, null).set(o);
-    }
+  public void angularBind(String name, Object o, String noteId) throws TException {
+    z.angularBind(name, o, noteId);
   }
 
   /**
@@ -262,100 +251,56 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
    * @param name name of the variable
    * @param o value
    */
-  public void angularBind(String name, Object o) {
+  public void angularBind(String name, Object o) throws TException {
     angularBind(name, o, interpreterContext.getNoteId());
   }
 
-  /*------------------------------------------RUN----------------------------------------*/
+  /**
+   * Run paragraph by id.
+   */
   @ZeppelinApi
-  public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
-      String paragraphId, InterpreterContext interpreterContext) {
-    RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
-    if (remoteWorksController != null) {
-      return remoteWorksController.getRemoteContextRunner(noteId, paragraphId);
-    }
-    return new LinkedList<InterpreterContextRunner>();
-  }
-
-  @ZeppelinApi
-  public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
-      InterpreterContext interpreterContext) {
-    RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
-    if (remoteWorksController != null) {
-      return remoteWorksController.getRemoteContextRunner(noteId);
-    }
-    return new LinkedList<InterpreterContextRunner>();
+  public void run(String noteId, String paragraphId) throws IOException {
+    z.run(noteId, paragraphId);
   }
 
   /**
    * Run paragraph by id.
    */
   @ZeppelinApi
-  public void run(String noteId, String paragraphId) {
-    run(noteId, paragraphId, interpreterContext);
+  public void run(String paragraphId) throws IOException {
+    z.run(paragraphId);
   }
 
   /**
    * Run paragraph by id.
    */
   @ZeppelinApi
-  public void run(String paragraphId) {
-    String noteId = interpreterContext.getNoteId();
-    run(noteId, paragraphId, interpreterContext);
+  public void run(String noteId, String paragraphId, InterpreterContext context)
+      throws IOException {
+    z.run(noteId, paragraphId, context);
   }
 
-  /**
-   * Run paragraph by id.
-   */
-  @ZeppelinApi
-  public void run(String noteId, String paragraphId, InterpreterContext context) {
-    if (paragraphId.equals(context.getParagraphId())) {
-      throw new RuntimeException("Can not run current Paragraph");
-    }
-    List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, paragraphId,
-        context);
-    if (runners.size() <= 0) {
-      throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size());
-    }
-    for (InterpreterContextRunner r : runners) {
-      r.run();
-    }
+  public void runNote(String noteId) throws IOException {
+    z.runNote(noteId);
   }
 
-  public void runNote(String noteId) {
-    runNote(noteId, interpreterContext);
-  }
-
-  public void runNote(String noteId, InterpreterContext context) {
-    String runningNoteId = context.getNoteId();
-    String runningParagraphId = context.getParagraphId();
-    List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
-
-    if (runners.size() <= 0) {
-      throw new RuntimeException("Note " + noteId + " not found " + runners.size());
-    }
-
-    for (InterpreterContextRunner r : runners) {
-      if (r.getNoteId().equals(runningNoteId) && r.getParagraphId().equals(runningParagraphId)) {
-        continue;
-      }
-      r.run();
-    }
+  public void runNote(String noteId, InterpreterContext context) throws IOException {
+    z.runNote(noteId, context);
   }
 
   /**
    * Run all paragraphs. except this.
    */
   @ZeppelinApi
-  public void runAll() {
-    runAll(interpreterContext);
+  public void runAll() throws IOException {
+    z.runAll(interpreterContext);
   }
 
   /**
    * Run all paragraphs. except this.
    */
   @ZeppelinApi
-  public void runAll(InterpreterContext context) {
-    runNote(context.getNoteId());
+  public void runAll(InterpreterContext context) throws IOException {
+    z.runNote(context.getNoteId());
   }
 }

groovy/src/main/java/org/apache/zeppelin/groovy/GroovyZeppelinContext.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.groovy;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ZeppelinContext for Groovy
+ */
+public class GroovyZeppelinContext extends BaseZeppelinContext {
+
+  public GroovyZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+    super(hooks, maxResult);
+  }
+
+  @Override
+  public Map<String, String> getInterpreterClassMap() {
+    return null;
+  }
+
+  @Override
+  public List<Class> getSupportedClasses() {
+    return null;
+  }
+
+  @Override
+  protected String showData(Object obj) {
+    return null;
+  }
+}

helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java

@@ -17,18 +17,17 @@
 
 package org.apache.zeppelin.helium;
 
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+
+import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-
 /**
  * Dummy interpreter to support development mode for Zeppelin app
  */
@@ -36,6 +35,7 @@ public class DevInterpreter extends Interpreter {
 
   private InterpreterEvent interpreterEvent;
   private InterpreterContext context;
+  private DevZeppelinContext z;
 
   public static boolean isInterpreterName(String replName) {
     return replName.equals("dev");
@@ -59,6 +59,7 @@ public class DevInterpreter extends Interpreter {
 
   @Override
   public void open() {
+    this.z = new DevZeppelinContext(null, 1000);
   }
 
   @Override
@@ -66,10 +67,10 @@ public class DevInterpreter extends Interpreter {
   }
 
   public void rerun() {
-    for (InterpreterContextRunner r : context.getRunners()) {
-      if (context.getParagraphId().equals(r.getParagraphId())) {
-        r.run();
-      }
+    try {
+      z.run(context.getParagraphId());
+    } catch (IOException e) {
+      throw new RuntimeException("Fail to rerun", e);
     }
   }
 
@@ -77,6 +78,7 @@ public class DevInterpreter extends Interpreter {
   public InterpreterResult interpret(String st, InterpreterContext context)
       throws InterpreterException {
     this.context = context;
+    this.z.setInterpreterContext(context);
     try {
       return interpreterEvent.interpret(st, context);
     } catch (Exception e) {

helium-dev/src/main/java/org/apache/zeppelin/helium/DevZeppelinContext.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.helium;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ZeppelinContext for DevInterpreter
+ */
+public class DevZeppelinContext extends BaseZeppelinContext {
+  public DevZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+    super(hooks, maxResult);
+  }
+
+  @Override
+  public Map<String, String> getInterpreterClassMap() {
+    return null;
+  }
+
+  @Override
+  public List<Class> getSupportedClasses() {
+    return null;
+  }
+
+  @Override
+  protected String showData(Object obj) {
+    return null;
+  }
+}

helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinApplicationDevServer.java

@@ -138,7 +138,7 @@ public class ZeppelinApplicationDevServer extends ZeppelinDevServer {
   protected InterpreterOutput createInterpreterOutput(
       final String noteId, final String paragraphId) {
     if (out == null) {
-      final RemoteInterpreterEventClient eventClient = getEventClient();
+      final RemoteInterpreterEventClient eventClient = getIntpEventClient();
       try {
         out = new InterpreterOutput(new InterpreterOutputListener() {
           @Override

helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java

@@ -39,7 +39,7 @@ public class ZeppelinDevServer extends
   private DevInterpreter interpreter = null;
   private InterpreterOutput out;
   public ZeppelinDevServer(int port) throws TException, IOException {
-    super(null, port, ":");
+    super(null, port, null, ":");
   }
 
   @Override
@@ -68,7 +68,7 @@ public class ZeppelinDevServer extends
   protected InterpreterOutput createInterpreterOutput(
       final String noteId, final String paragraphId) {
     if (out == null) {
-      final RemoteInterpreterEventClient eventClient = getEventClient();
+      final RemoteInterpreterEventClient eventClient = getIntpEventClient();
       try {
         out = new InterpreterOutput(new InterpreterOutputListener() {
           @Override

ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java

@@ -16,14 +16,13 @@
  */
 package org.apache.zeppelin.ignite;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import org.apache.ignite.Ignite;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -31,8 +30,8 @@ import org.junit.Test;
 import java.util.Collections;
 import java.util.Properties;
 
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests for Apache Ignite interpreter ({@link IgniteInterpreter}).
@@ -40,9 +39,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
 public class IgniteInterpreterTest {
   private static final String HOST = "127.0.0.1:47500..47509";
 
-  private static final InterpreterContext INTP_CONTEXT =
-      new InterpreterContext(null, null, null, null, null, null, null, null, null, null,
-              null, null, null);
+  private static final InterpreterContext INTP_CONTEXT = InterpreterContext.builder().build();
 
   private IgniteInterpreter intp;
   private Ignite ignite;

ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java

@@ -16,8 +16,6 @@
  */
 package org.apache.zeppelin.ignite;
 
-import static org.junit.Assert.assertEquals;
-
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -25,6 +23,11 @@ import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -32,11 +35,7 @@ import org.junit.Test;
 import java.util.Collections;
 import java.util.Properties;
 
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import static org.junit.Assert.assertEquals;
 
 /**
  * Tests for Apache Ignite SQL interpreter ({@link IgniteSqlInterpreter}).
@@ -44,9 +43,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 public class IgniteSqlInterpreterTest {
   private static final String HOST = "127.0.0.1:47500..47509";
 
-  private static final InterpreterContext INTP_CONTEXT =
-      new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null,
-              null, null);
+  private static final InterpreterContext INTP_CONTEXT = InterpreterContext.builder().build();
 
   private Ignite ignite;
   private IgniteSqlInterpreter intp;

interpreter-parent/pom.xml

@@ -38,7 +38,6 @@
         <groupId>${project.groupId}</groupId>
         <artifactId>zeppelin-interpreter</artifactId>
         <version>${project.version}</version>
-        <scope>provided</scope>
       </dependency>
 
       <dependency>
@@ -79,7 +78,7 @@
           <artifactId>maven-dependency-plugin</artifactId>
           <executions>
             <execution>
-              <id>copy-interpreter-dependencies</id>
+              <id>copy-dependencies</id>
               <phase>package</phase>
               <goals>
                 <goal>copy-dependencies</goal>

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterInterpolationTest.java

@@ -69,8 +69,11 @@ public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
                 "('mou', 'mouse');");
     resourcePool = new LocalResourcePool("JdbcInterpolationTest");
 
-    interpreterContext = new InterpreterContext("", "1", null, "", "",
-            new AuthenticationInfo("testUser"), null, null, null, null, resourcePool, null, null);
+    interpreterContext = InterpreterContext.builder()
+        .setParagraphId("paragraph_1")
+        .setAuthenticationInfo(new AuthenticationInfo("testUser"))
+        .setResourcePool(resourcePool)
+        .build();
   }
 
   @Test

jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java

@@ -100,8 +100,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
             "insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
     insertStatement.setString(1, null);
     insertStatement.execute();
-    interpreterContext = new InterpreterContext("", "1", null, "", "",
-        new AuthenticationInfo("testUser"), null, null, null, null, null, null, null);
+    interpreterContext = InterpreterContext.builder()
+        .setAuthenticationInfo(new AuthenticationInfo("testUser"))
+        .build();
   }
 
 
@@ -433,8 +434,10 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     // user1 runs jdbc1
     jdbc1.open();
-    InterpreterContext ctx1 = new InterpreterContext("", "1", "jdbc1", "", "", user1Credential,
-            null, null, null, null, null, null, null);
+    InterpreterContext ctx1 = InterpreterContext.builder()
+        .setAuthenticationInfo(user1Credential)
+        .setReplName("jdbc1")
+        .build();
     jdbc1.interpret("", ctx1);
 
     JDBCUserConfigurations user1JDBC1Conf = jdbc1.getJDBCConfiguration("user1");
@@ -444,8 +447,10 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     // user1 runs jdbc2
     jdbc2.open();
-    InterpreterContext ctx2 = new InterpreterContext("", "1", "jdbc2", "", "", user1Credential,
-            null, null, null, null, null, null, null);
+    InterpreterContext ctx2 = InterpreterContext.builder()
+        .setAuthenticationInfo(user1Credential)
+        .setReplName("jdbc2")
+        .build();
     jdbc2.interpret("", ctx2);
 
     JDBCUserConfigurations user1JDBC2Conf = jdbc2.getJDBCConfiguration("user1");
@@ -455,8 +460,10 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     // user2 runs jdbc1
     jdbc1.open();
-    InterpreterContext ctx3 = new InterpreterContext("", "1", "jdbc1", "", "", user2Credential,
-            null, null, null, null, null, null, null);
+    InterpreterContext ctx3 = InterpreterContext.builder()
+        .setAuthenticationInfo(user2Credential)
+        .setReplName("jdbc1")
+        .build();
     jdbc1.interpret("", ctx3);
 
     JDBCUserConfigurations user2JDBC1Conf = jdbc1.getJDBCConfiguration("user2");
@@ -466,8 +473,10 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
 
     // user2 runs jdbc2
     jdbc2.open();
-    InterpreterContext ctx4 = new InterpreterContext("", "1", "jdbc2", "", "", user2Credential,
-            null, null, null, null, null, null, null);
+    InterpreterContext ctx4 = InterpreterContext.builder()
+        .setAuthenticationInfo(user2Credential)
+        .setReplName("jdbc2")
+        .build();
     jdbc2.interpret("", ctx4);
 
     JDBCUserConfigurations user2JDBC2Conf = jdbc2.getJDBCConfiguration("user2");

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java

@@ -94,6 +94,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
   private String livyURL;
   private int sessionCreationTimeout;
   private int pullStatusInterval;
+  private int maxLogLines;
   protected boolean displayAppInfo;
   private boolean restartDeadSession;
   protected LivyVersion livyVersion;
@@ -119,6 +120,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
         property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
     this.pullStatusInterval = Integer.parseInt(
         property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
+    this.maxLogLines = Integer.parseInt(property.getProperty("zeppelin.livy.maxLogLines",
+        "1000"));
     this.restTemplate = createRestTemplate();
     if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) {
       String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";");
@@ -338,7 +341,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
         if ((System.currentTimeMillis() - start) / 1000 > sessionCreationTimeout) {
           String msg = "The creation of session " + sessionInfo.id + " is timeout within "
               + sessionCreationTimeout + " seconds, appId: " + sessionInfo.appId
-              + ", log: " + sessionInfo.log;
+              + ", log:\n" + StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
           throw new LivyException(msg);
         }
         Thread.sleep(pullStatusInterval);
@@ -347,7 +350,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
             sessionInfo.appId);
         if (sessionInfo.isFinished()) {
           String msg = "Session " + sessionInfo.id + " is finished, appId: " + sessionInfo.appId
-              + ", log: " + sessionInfo.log;
+              + ", log:\n" + StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
           throw new LivyException(msg);
         }
       }
@@ -362,6 +365,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
     return SessionInfo.fromJson(callRestAPI("/sessions/" + sessionId, "GET"));
   }
 
+  private SessionLog getSessionLog(int sessionId) throws LivyException {
+    return SessionLog.fromJson(callRestAPI("/sessions/" + sessionId + "/log?size=" + maxLogLines,
+        "GET"));
+  }
+
   public InterpreterResult interpret(String code,
                                      String paragraphId,
                                      boolean displayAppInfo,
@@ -822,6 +830,20 @@ public abstract class BaseLivyInterpreter extends Interpreter {
     }
   }
 
+  private static class SessionLog {
+    public int id;
+    public int from;
+    public int size;
+    public List<String> log;
+
+    SessionLog() {
+    }
+
+    public static SessionLog fromJson(String json) {
+      return gson.fromJson(json, SessionLog.class);
+    }
+  }
+
   static class ExecuteRequest {
     public final String code;
     public final String kind;

livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java

@@ -17,16 +17,7 @@
 
 package org.apache.zeppelin.livy;
 
-import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
-
 import org.apache.commons.lang.StringUtils;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
@@ -39,7 +30,12 @@ import org.apache.zeppelin.interpreter.ResultMessages;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.apache.zeppelin.user.AuthenticationInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
 
 /**
  * Livy SparkSQL Interpreter for Zeppelin.
@@ -78,20 +74,9 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
     // As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession
     // to judge whether it is using spark2.
     try {
-      InterpreterContext context = new InterpreterContext(
-          "noteId",
-          "paragraphId",
-          "replName",
-          "paragraphTitle",
-          "paragraphText",
-          new AuthenticationInfo(),
-          new HashMap<String, Object>(),
-          new GUI(),
-          new GUI(),
-          null,
-          null,
-          null,
-          new InterpreterOutput(null));
+      InterpreterContext context = InterpreterContext.builder()
+          .setInterpreterOut(new InterpreterOutput(null))
+          .build();
       InterpreterResult result = sparkInterpreter.interpret("spark", context);
       if (result.code() == InterpreterResult.Code.SUCCESS &&
           result.message().get(0).getData().contains("org.apache.spark.sql.SparkSession")) {

livy/src/main/resources/interpreter-setting.json

@@ -97,6 +97,12 @@
         "description": "The interval for checking paragraph execution status",
         "type": "number"
       },
+      "zeppelin.livy.maxLogLines": {
+        "propertyName": "zeppelin.livy.maxLogLines",
+        "defaultValue": "1000",
+        "description": "Max number of lines of logs",
+        "type": "number"
+      },
       "livy.spark.jars.packages": {
         "propertyName": "livy.spark.jars.packages",
         "defaultValue": "",
@@ -105,13 +111,13 @@
       },
       "zeppelin.livy.displayAppInfo": {
         "propertyName": "zeppelin.livy.displayAppInfo",
-        "defaultValue": "true",
+        "defaultValue": true,
         "description": "Whether display app info",
         "type": "checkbox"
       },
       "zeppelin.livy.restart_dead_session": {
         "propertyName": "zeppelin.livy.restart_dead_session",
-        "defaultValue": "false",
+        "defaultValue": false,
         "description": "Whether restart a dead session",
         "type": "checkbox"
       }

livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java

@@ -17,23 +17,9 @@
 
 package org.apache.zeppelin.livy;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.livy.test.framework.Cluster;
 import org.apache.livy.test.framework.Cluster$;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Properties;
-
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
@@ -44,6 +30,19 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
 
 public class LivyInterpreterIT {
   private static final Logger LOGGER = LoggerFactory.getLogger(LivyInterpreterIT.class);
@@ -98,8 +97,12 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
-        "title", "text", authInfo, null, null, null, null, null, null, output);
+    InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
     sparkInterpreter.open();
 
     LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
@@ -127,8 +130,13 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
-        "title", "text", authInfo, null, null, null, null, null, null, output);
+    final InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
+    ;
 
     InterpreterResult result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -226,8 +234,12 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
-        "title", "text", authInfo, null, null, null, null, null, null, output);
+    final InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
 
     InterpreterResult result = null;
     // test DataFrame api
@@ -312,8 +324,8 @@ public class LivyInterpreterIT {
     if (!isSpark2) {
       result = sparkInterpreter.interpret(
           "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
-                  + ".toDF(\"col_1\", \"col_2\")\n"
-                  + "df.collect()", context);
+              + ".toDF(\"col_1\", \"col_2\")\n"
+              + "df.collect()", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(1, result.message().size());
       assertTrue(result.message().get(0).getData()
@@ -321,8 +333,8 @@ public class LivyInterpreterIT {
     } else {
       result = sparkInterpreter.interpret(
           "val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
-                  + ".toDF(\"col_1\", \"col_2\")\n"
-                  + "df.collect()", context);
+              + ".toDF(\"col_1\", \"col_2\")\n"
+              + "df.collect()", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(1, result.message().size());
       assertTrue(result.message().get(0).getData()
@@ -331,7 +343,7 @@ public class LivyInterpreterIT {
     sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
     // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
     result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'",
-            context);
+        context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
     assertEquals("col_1\tcol_2\n12characters12cha...\t20", result.message().get(0).getData());
@@ -349,8 +361,12 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId",
-            "livy.pyspark", "title", "text", authInfo, null, null, null, null, null, null, output);
+    final InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
     pysparkInterpreter.open();
 
     // test traceback msg
@@ -359,7 +375,7 @@ public class LivyInterpreterIT {
       // for livy version >=0.3 , input some erroneous spark code, check the shown result is more
       // than one line
       InterpreterResult result = pysparkInterpreter.interpret(
-              "sc.parallelize(wrongSyntax(1, 2)).count()", context);
+          "sc.parallelize(wrongSyntax(1, 2)).count()", context);
       assertEquals(InterpreterResult.Code.ERROR, result.code());
       assertTrue(result.message().get(0).getData().split("\n").length > 1);
       assertTrue(result.message().get(0).getData().contains("Traceback"));
@@ -465,7 +481,7 @@ public class LivyInterpreterIT {
 
   @Test
   public void testSparkInterpreterWithDisplayAppInfo_StringWithoutTruncation()
-          throws InterpreterException {
+      throws InterpreterException {
     if (!checkPreCondition()) {
       return;
     }
@@ -482,8 +498,12 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
-        "title", "text", authInfo, null, null, null, null, null, null, output);
+    InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
     sparkInterpreter.open();
 
     LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties2);
@@ -495,7 +515,8 @@ public class LivyInterpreterIT {
       InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(2, result.message().size());
-      assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
+      // check yarn appId and ensure it is not null
+      assertTrue(result.message().get(1).getData().contains("Spark Application Id: application_"));
 
       // html output
       String htmlCode = "println(\"%html <h1> hello </h1>\")";
@@ -514,8 +535,8 @@ public class LivyInterpreterIT {
       if (!isSpark2) {
         result = sparkInterpreter.interpret(
             "val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
-                    + ".toDF(\"col_1\", \"col_2\")\n"
-                    + "df.collect()", context);
+                + ".toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
         assertEquals(InterpreterResult.Code.SUCCESS, result.code());
         assertEquals(2, result.message().size());
         assertTrue(result.message().get(0).getData()
@@ -523,8 +544,8 @@ public class LivyInterpreterIT {
       } else {
         result = sparkInterpreter.interpret(
             "val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
-                    + ".toDF(\"col_1\", \"col_2\")\n"
-                    + "df.collect()", context);
+                + ".toDF(\"col_1\", \"col_2\")\n"
+                + "df.collect()", context);
         assertEquals(InterpreterResult.Code.SUCCESS, result.code());
         assertEquals(2, result.message().size());
         assertTrue(result.message().get(0).getData()
@@ -533,7 +554,7 @@ public class LivyInterpreterIT {
       sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
       // test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
       result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'",
-              context);
+          context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
       assertEquals("col_1\tcol_2\n12characters12characters\t20", result.message().get(0).getData());
@@ -561,8 +582,12 @@ public class LivyInterpreterIT {
     AuthenticationInfo authInfo = new AuthenticationInfo("user1");
     MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
     InterpreterOutput output = new InterpreterOutput(outputListener);
-    final InterpreterContext context = new InterpreterContext("noteId", "paragraphId",
-            "livy.sparkr", "title", "text", authInfo, null, null, null, null, null, null, output);
+    final InterpreterContext context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(authInfo)
+        .setInterpreterOut(output)
+        .build();
     sparkRInterpreter.open();
 
     try {
@@ -637,8 +662,12 @@ public class LivyInterpreterIT {
       AuthenticationInfo authInfo = new AuthenticationInfo("user1");
       MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
       InterpreterOutput output = new InterpreterOutput(outputListener);
-      InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql",
-          "title", "text", authInfo, null, null, null, null, null, null, output);
+      InterpreterContext context = InterpreterContext.builder()
+          .setNoteId("noteId")
+          .setParagraphId("paragraphId")
+          .setAuthenticationInfo(authInfo)
+          .setInterpreterOut(output)
+          .build();
 
       String p1 = IOUtils.toString(getClass().getResourceAsStream("/livy_tutorial_1.scala"));
       InterpreterResult result = sparkInterpreter.interpret(p1, context);
@@ -695,15 +724,19 @@ public class LivyInterpreterIT {
       AuthenticationInfo authInfo = new AuthenticationInfo("user1");
       MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
       InterpreterOutput output = new InterpreterOutput(outputListener);
-      InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql",
-          "title", "text", authInfo, null, null, null, null, null, null, output);
+      InterpreterContext context = InterpreterContext.builder()
+          .setNoteId("noteId")
+          .setParagraphId("paragraphId")
+          .setAuthenticationInfo(authInfo)
+          .setInterpreterOut(output)
+          .build();
       // detect spark version
       InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(1, result.message().size());
 
       boolean isSpark2 = isSpark2((BaseLivyInterpreter) sparkInterpreter.getInnerInterpreter(),
-              context);
+          context);
 
       if (!isSpark2) {
         result = sparkInterpreter.interpret(
@@ -717,7 +750,7 @@ public class LivyInterpreterIT {
 
         // access table from pyspark
         result = pysparkInterpreter.interpret("sqlContext.sql(\"select * from df\").show()",
-                context);
+            context);
         assertEquals(InterpreterResult.Code.SUCCESS, result.code());
         assertEquals(1, result.message().size());
         assertTrue(result.message().get(0).getData()
@@ -729,7 +762,7 @@ public class LivyInterpreterIT {
 
         // access table from sparkr
         result = sparkRInterpreter.interpret("head(sql(sqlContext, \"select * from df\"))",
-                context);
+            context);
         assertEquals(InterpreterResult.Code.SUCCESS, result.code());
         assertEquals(1, result.message().size());
         assertTrue(result.message().get(0).getData().contains("col_1 col_2\n1 hello    20"));
@@ -764,11 +797,11 @@ public class LivyInterpreterIT {
       // test plotting of python
       result = pysparkInterpreter.interpret(
           "import matplotlib.pyplot as plt\n" +
-          "plt.switch_backend('agg')\n" +
-          "data=[1,2,3,4]\n" +
-          "plt.figure()\n" +
-          "plt.plot(data)\n" +
-          "%matplot plt", context);
+              "plt.switch_backend('agg')\n" +
+              "data=[1,2,3,4]\n" +
+              "plt.figure()\n" +
+              "plt.plot(data)\n" +
+              "%matplot plt", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertEquals(1, result.message().size());
       assertEquals(InterpreterResult.Type.IMG, result.message().get(0).getType());

neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java

@@ -16,11 +16,14 @@
  */
 package org.apache.zeppelin.graph.neo4j;
 
-import static org.junit.Assert.assertEquals;
-
 import com.google.gson.Gson;
-
 import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.graph.GraphResult;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -32,23 +35,10 @@ import org.neo4j.harness.ServerControls;
 import org.neo4j.harness.TestServerBuilders;
 
 import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterOutput;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.interpreter.graph.GraphResult;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
+import static org.junit.Assert.assertEquals;
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class Neo4jCypherInterpreterTest {
@@ -91,15 +81,9 @@ public class Neo4jCypherInterpreterTest {
     p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString());
     p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50");
     interpreter = new Neo4jCypherInterpreter(p);
-    context = new InterpreterContext("note", "id", null, "title", "text",
-            new AuthenticationInfo(),
-            new HashMap<String, Object>(),
-            new GUI(),
-            new GUI(),
-            new AngularObjectRegistry(new InterpreterGroup().getId(), null),
-            new LocalResourcePool("id"),
-            new LinkedList<InterpreterContextRunner>(),
-            new InterpreterOutput(null));
+    context = InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .build();;
   }
 
   @After

pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java

@@ -43,8 +43,7 @@ public class PigInterpreterSparkTest {
     properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
     pigInterpreter = new PigInterpreter(properties);
     pigInterpreter.open();
-    context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
-        null, null, null, null);
+    context = InterpreterContext.builder().setParagraphId("paragraphId").build();
 
   }
   @After

pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java

@@ -46,8 +46,7 @@ public class PigInterpreterTest {
     properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
     pigInterpreter = new PigInterpreter(properties);
     pigInterpreter.open();
-    context = new InterpreterContext(null, "paragraph_id", null, null, null,
-        null, null, null, null, null, null, null, null);
+    context = InterpreterContext.builder().setParagraphId("paragraphId").build();
   }
 
   @After

pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java

@@ -47,8 +47,7 @@ public class PigInterpreterTezTest {
     properties.put("tez.queue.name", "test");
     pigInterpreter = new PigInterpreter(properties);
     pigInterpreter.open();
-    context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
-        null, null, null, null);
+    context = InterpreterContext.builder().setParagraphId("paragraphId").build();
 
   }
   @After

pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java

@@ -66,8 +66,7 @@ public class PigQueryInterpreterTest {
     pigInterpreter.open();
     pigQueryInterpreter.open();
 
-    context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
-        null, null, null, null);
+    context = InterpreterContext.builder().setParagraphId("paragraphId").build();
   }
 
   @After

pom.xml

@@ -101,7 +101,7 @@
     <!-- common library versions -->
     <slf4j.version>1.7.10</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
-    <libthrift.version>0.9.2</libthrift.version>
+    <libthrift.version>0.9.3</libthrift.version>
     <gson.version>2.2</gson.version>
     <gson-extras.version>0.2.1</gson-extras.version>
     <jetty.version>9.2.15.v20160210</jetty.version>

python/pom.xml

@@ -214,6 +214,13 @@
         <artifactId>maven-resources-plugin</artifactId>
       </plugin>
 
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <configuration>
+          <skip>false</skip>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

python/src/main/java/org/apache/zeppelin/python/IPythonClient.java

@@ -38,10 +38,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
 import java.security.SecureRandom;
+import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 

python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java

@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
@@ -26,7 +26,6 @@ import org.apache.commons.exec.ExecuteWatchdog;
 import org.apache.commons.exec.LogOutputStream;
 import org.apache.commons.exec.PumpStreamHandler;
 import org.apache.commons.exec.environment.EnvironmentUtils;
-import org.apache.commons.httpclient.util.ExceptionUtil;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
@@ -60,8 +59,6 @@ import java.io.InputStream;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -157,7 +154,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
   /**
    * non-empty return value mean the errors when checking ipython prerequisite.
    * empty value mean IPython prerequisite is meet.
-   * 
+   *
    * @param pythonExec
    * @return
    */
@@ -398,11 +395,11 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
     LOGGER.warn("Exception happens in Python Process", e);
   }
 
-  private static class ProcessLogOutputStream extends LogOutputStream {
+  static class ProcessLogOutputStream extends LogOutputStream {
 
     private Logger logger;
 
-    public ProcessLogOutputStream(Logger logger) {
+    ProcessLogOutputStream(Logger logger) {
       this.logger = logger;
     }
 

python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java

@@ -1,31 +1,44 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.zeppelin.python;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
-import java.util.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -161,7 +174,7 @@ public class PythonCondaInterpreter extends Interpreter {
     PythonInterpreter python = null;
     Interpreter p =
         getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
-    return (PythonInterpreter) ((LazyOpenInterpreter)p).getInnerInterpreter();
+    return (PythonInterpreter) ((LazyOpenInterpreter) p).getInnerInterpreter();
   }
 
   public static String runCondaCommandForTextOutput(String title, List<String> commands)
@@ -410,7 +423,7 @@ public class PythonCondaInterpreter extends Interpreter {
         BufferedReader br = new BufferedReader(isr);
         String line = null;
         long startTime = System.currentTimeMillis();
-        while ( (line = br.readLine()) != null) {
+        while ((line = br.readLine()) != null) {
           output.append(line + "\n");
           // logging per 5 seconds
           if ((System.currentTimeMillis() - startTime) > 5000) {
@@ -428,7 +441,7 @@ public class PythonCondaInterpreter extends Interpreter {
     }
   }
 
-  public static String runCommand(String ... command)
+  public static String runCommand(String... command)
       throws IOException, InterruptedException {
 
     List<String> list = new ArrayList<>(command.length);

python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java

@@ -1,27 +1,37 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.zeppelin.python;
 
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
+import org.apache.zeppelin.interpreter.WrappedInterpreter;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.file.Paths;
 import java.util.Properties;
 import java.util.regex.Matcher;

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -47,11 +47,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import py4j.GatewayServer;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -62,13 +60,13 @@ import java.util.Properties;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- *  Interpreter for Python, it is the first implementation of interpreter for Python, so with less
- *  features compared to IPythonInterpreter, but requires less prerequisites than
- *  IPythonInterpreter, only python installation is required.
+ * Interpreter for Python, it is the first implementation of interpreter for Python, so with less
+ * features compared to IPythonInterpreter, but requires less prerequisites than
+ * IPythonInterpreter, only python installation is required.
  */
 public class PythonInterpreter extends Interpreter implements ExecuteResultHandler {
   private static final Logger LOGGER = LoggerFactory.getLogger(PythonInterpreter.class);
-  private static final int MAX_TIMEOUT_SEC = 10;
+  private static final int MAX_TIMEOUT_SEC = 30;
 
   private GatewayServer gatewayServer;
   private DefaultExecutor executor;
@@ -141,7 +139,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
         InetAddress.getByName(serverAddress),
         GatewayServer.DEFAULT_CONNECT_TIMEOUT,
         GatewayServer.DEFAULT_READ_TIMEOUT,
-        (List) null);;
+        (List) null);
+    ;
     gatewayServer.start();
     LOGGER.info("Starting GatewayServer at " + serverAddress + ":" + port);
 
@@ -292,10 +291,16 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
   public class PythonInterpretRequest {
     public String statements;
     public boolean isForCompletion;
+    public boolean isCallHooks;
 
     public PythonInterpretRequest(String statements, boolean isForCompletion) {
+      this(statements, isForCompletion, true);
+    }
+
+    public PythonInterpretRequest(String statements, boolean isForCompletion, boolean isCallHooks) {
       this.statements = statements;
       this.isForCompletion = isForCompletion;
+      this.isCallHooks = isCallHooks;
     }
 
     public String statements() {
@@ -305,6 +310,10 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     public boolean isForCompletion() {
       return isForCompletion;
     }
+
+    public boolean isCallHooks() {
+      return isCallHooks;
+    }
   }
 
   // called by Python Process
@@ -367,6 +376,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
         try {
           statementFinishedNotifier.wait(1000);
         } catch (InterruptedException e) {
+          // ignore this exception
         }
       }
     }
@@ -522,7 +532,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     }
 
     List<InterpreterCompletion> results = new LinkedList<>();
-    for (String name: completionList) {
+    for (String name : completionList) {
       results.add(new InterpreterCompletion(name, name, StringUtils.EMPTY));
     }
     return results;
@@ -538,8 +548,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     String completionScriptText = "";
     try {
       completionScriptText = text.substring(0, cursor);
-    }
-    catch (Exception e) {
+    } catch (Exception e) {
       LOGGER.error(e.toString());
       return null;
     }
@@ -558,13 +567,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
 
     if (completionStartPosition == completionEndPosition) {
       completionStartPosition = 0;
-    }
-    else
-    {
+    } else {
       completionStartPosition = completionEndPosition - completionStartPosition;
     }
     resultCompletionText = completionScriptText.substring(
-        completionStartPosition , completionEndPosition);
+        completionStartPosition, completionEndPosition);
 
     return resultCompletionText;
   }
@@ -602,7 +609,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     String bootstrapCode =
         IOUtils.toString(getClass().getClassLoader().getResourceAsStream(resourceName));
     try {
-      InterpreterResult result = interpret(bootstrapCode, InterpreterContext.get());
+      // Add hook explicitly, otherwise python will fail to execute the statement
+      InterpreterResult result = interpret(bootstrapCode + "\n" + "__zeppelin__._displayhook()",
+          InterpreterContext.get());
       if (result.code() != Code.SUCCESS) {
         throw new IOException("Fail to run bootstrap script: " + resourceName);
       }

python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java

@@ -1,25 +1,22 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
-import java.io.IOException;
-import java.util.Properties;
-
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
@@ -29,9 +26,12 @@ import org.apache.zeppelin.interpreter.WrappedInterpreter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.util.Properties;
+
 /**
  * SQL over Pandas DataFrame interpreter for %python group
- *
+ * <p>
  * Match experience of %sparpk.sql over Spark DataFrame
  */
 public class PythonInterpreterPandasSql extends Interpreter {
@@ -90,7 +90,7 @@ public class PythonInterpreterPandasSql extends Interpreter {
     Interpreter python = getPythonInterpreter();
 
     return python.interpret(
-            "__zeppelin__.show(pysqldf('" + st + "'))\n__zeppelin__._displayhook()", context);
+        "__zeppelin__.show(pysqldf('" + st + "'))\n__zeppelin__._displayhook()", context);
   }
 
   @Override

python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java

@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 

python/src/main/resources/python/zeppelin_python.py

@@ -111,12 +111,18 @@ while True :
 
     # Get post-execute hooks
     try:
-      global_hook = intp.getHook('post_exec_dev')
+      if req.isCallHooks():
+        global_hook = intp.getHook('post_exec_dev')
+      else:
+        global_hook = None
     except:
       global_hook = None
 
     try:
-      user_hook = __zeppelin__.getHook('post_exec')
+      if req.isCallHooks():
+        user_hook = __zeppelin__.getHook('post_exec')
+      else:
+        user_hook = None
     except:
       user_hook = None
 
@@ -133,7 +139,6 @@ while True :
       to_run_hooks = []
       if (nhooks > 0):
         to_run_hooks = code.body[-nhooks:]
-
       to_run_exec, to_run_single = (code.body[:-(nhooks + 1)],
                                     [code.body[-(nhooks + 1)]])
       try:

python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java

@@ -17,7 +17,6 @@
 
 package org.apache.zeppelin.python;
 
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.display.ui.CheckBox;
 import org.apache.zeppelin.display.ui.Select;
 import org.apache.zeppelin.display.ui.TextBox;
@@ -28,19 +27,18 @@ import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
 
 import static junit.framework.TestCase.assertTrue;
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
 
 public abstract class BasePythonInterpreterTest {
 
@@ -58,10 +56,12 @@ public abstract class BasePythonInterpreterTest {
   public void testPythonBasics() throws InterpreterException, InterruptedException, IOException {
 
     InterpreterContext context = getInterpreterContext();
-    InterpreterResult result = interpreter.interpret("import sys\nprint(sys.version[0])", context);
+    InterpreterResult result =
+        interpreter.interpret("import sys\nprint(sys.version[0])", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     Thread.sleep(100);
-    List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
+    List<InterpreterResultMessage> interpreterResultMessages =
+        context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
 
     // single output without print
@@ -111,7 +111,7 @@ public abstract class BasePythonInterpreterTest {
 
     // assignment
     context = getInterpreterContext();
-    result = interpreter.interpret("abc=1",context);
+    result = interpreter.interpret("abc=1", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
@@ -119,7 +119,8 @@ public abstract class BasePythonInterpreterTest {
 
     // if block
     context = getInterpreterContext();
-    result = interpreter.interpret("if abc > 0:\n\tprint('True')\nelse:\n\tprint('False')", context);
+    result =
+        interpreter.interpret("if abc > 0:\n\tprint('True')\nelse:\n\tprint('False')", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
@@ -143,7 +144,8 @@ public abstract class BasePythonInterpreterTest {
     if (interpreter instanceof IPythonInterpreter) {
       interpreterResultMessages = context.out.toInterpreterResultMessage();
       assertEquals(1, interpreterResultMessages.size());
-      assertTrue(interpreterResultMessages.get(0).getData().contains("name 'unknown' is not defined"));
+      assertTrue(interpreterResultMessages.get(0).getData().contains(
+          "name 'unknown' is not defined"));
     } else if (interpreter instanceof PythonInterpreter) {
       assertTrue(result.message().get(0).getData().contains("name 'unknown' is not defined"));
     }
@@ -165,14 +167,14 @@ public abstract class BasePythonInterpreterTest {
     context = getInterpreterContext();
     result = interpreter.interpret(
         "from __future__ import print_function\n" +
-        "def greet(name):\n" +
-        "    print('Hello', name)\n" +
-        "greet('Jack')", context);
+            "def greet(name):\n" +
+            "    print('Hello', name)\n" +
+            "greet('Jack')", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
-    assertEquals("Hello Jack\n",interpreterResultMessages.get(0).getData());
+    assertEquals("Hello Jack\n", interpreterResultMessages.get(0).getData());
 
     // ZEPPELIN-1114
     context = getInterpreterContext();
@@ -225,10 +227,12 @@ public abstract class BasePythonInterpreterTest {
   public void testZeppelinContext() throws InterpreterException, InterruptedException, IOException {
     // TextBox
     InterpreterContext context = getInterpreterContext();
-    InterpreterResult result = interpreter.interpret("z.input(name='text_1', defaultValue='value_1')", context);
+    InterpreterResult result =
+        interpreter.interpret("z.input(name='text_1', defaultValue='value_1')", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
+    List<InterpreterResultMessage> interpreterResultMessages =
+        context.out.toInterpreterResultMessage();
     assertTrue(interpreterResultMessages.get(0).getData().contains("'value_1'"));
     assertEquals(1, context.getGui().getForms().size());
     assertTrue(context.getGui().getForms().get("text_1") instanceof TextBox);
@@ -238,7 +242,8 @@ public abstract class BasePythonInterpreterTest {
 
     // Select
     context = getInterpreterContext();
-    result = interpreter.interpret("z.select(name='select_1', options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
+    result = interpreter.interpret("z.select(name='select_1'," +
+        " options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertEquals(1, context.getGui().getForms().size());
     assertTrue(context.getGui().getForms().get("select_1") instanceof Select);
@@ -250,7 +255,8 @@ public abstract class BasePythonInterpreterTest {
 
     // CheckBox
     context = getInterpreterContext();
-    result = interpreter.interpret("z.checkbox(name='checkbox_1', options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
+    result = interpreter.interpret("z.checkbox(name='checkbox_1'," +
+        "options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertEquals(1, context.getGui().getForms().size());
     assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox);
@@ -262,7 +268,8 @@ public abstract class BasePythonInterpreterTest {
 
     // Pandas DataFrame
     context = getInterpreterContext();
-    result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context);
+    result = interpreter.interpret("import pandas as pd\n" +
+        "df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
@@ -270,18 +277,21 @@ public abstract class BasePythonInterpreterTest {
     assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData());
 
     context = getInterpreterContext();
-    result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", context);
+    result = interpreter.interpret("import pandas as pd\n" +
+        "df = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals(2, interpreterResultMessages.size());
     assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType());
     assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData());
     assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
-    assertEquals("<font color=red>Results are limited by 3.</font>\n", interpreterResultMessages.get(1).getData());
+    assertEquals("<font color=red>Results are limited by 3.</font>\n",
+        interpreterResultMessages.get(1).getData());
 
     // z.show(matplotlib)
     context = getInterpreterContext();
-    result = interpreter.interpret("import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", context);
+    result = interpreter.interpret("import matplotlib.pyplot as plt\n" +
+        "data=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
@@ -289,7 +299,8 @@ public abstract class BasePythonInterpreterTest {
 
     // clear output
     context = getInterpreterContext();
-    result = interpreter.interpret("import time\nprint(\"Hello\")\ntime.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context);
+    result = interpreter.interpret("import time\nprint(\"Hello\")\n" +
+        "time.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context);
     assertEquals("%text world\n", context.out.getCurrentOutput().toString());
   }
 
@@ -299,33 +310,24 @@ public abstract class BasePythonInterpreterTest {
     String restoreCode = "z = __zeppelin__\n";
     String validCode = "z.input(\"test\")\n";
 
-    assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code());
-    assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(redefinitionCode, getInterpreterContext()).code());
-    assertEquals(InterpreterResult.Code.ERROR, interpreter.interpret(validCode, getInterpreterContext()).code());
-    assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(restoreCode, getInterpreterContext()).code());
-    assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code());
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        interpreter.interpret(validCode, getInterpreterContext()).code());
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        interpreter.interpret(redefinitionCode, getInterpreterContext()).code());
+    assertEquals(InterpreterResult.Code.ERROR,
+        interpreter.interpret(validCode, getInterpreterContext()).code());
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        interpreter.interpret(restoreCode, getInterpreterContext()).code());
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        interpreter.interpret(validCode, getInterpreterContext()).code());
   }
 
   protected InterpreterContext getInterpreterContext() {
-    return new InterpreterContext(
-        "noteId",
-        "paragraphId",
-        "replName",
-        "paragraphTitle",
-        "paragraphText",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        null,
-        null,
-        null,
-        new InterpreterOutput(null));
-  }
-
-  protected InterpreterContext getInterpreterContext(RemoteEventClient mockRemoteEventClient) {
-    InterpreterContext context = getInterpreterContext();
-    context.setClient(mockRemoteEventClient);
-    return context;
+    return InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
+        .build();
   }
 }

python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java

@@ -33,12 +33,10 @@ import java.util.Properties;
 
 import static junit.framework.TestCase.assertTrue;
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
 
 
 public class IPythonInterpreterTest extends BasePythonInterpreterTest {
 
-
   protected Properties initIntpProperties() {
     Properties properties = new Properties();
     properties.setProperty("zeppelin.python.maxResult", "3");
@@ -68,13 +66,15 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
   }
 
   @Test
-  public void testIPythonAdvancedFeatures() throws InterpreterException, InterruptedException, IOException {
+  public void testIPythonAdvancedFeatures()
+      throws InterpreterException, InterruptedException, IOException {
     // ipython help
     InterpreterContext context = getInterpreterContext();
     InterpreterResult result = interpreter.interpret("range?", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
+    List<InterpreterResultMessage> interpreterResultMessages =
+        context.out.toInterpreterResultMessage();
     assertTrue(interpreterResultMessages.get(0).getData().contains("range(stop)"));
 
     // timeit
@@ -113,10 +113,12 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
   public void testIPythonPlotting() throws InterpreterException, InterruptedException, IOException {
     // matplotlib
     InterpreterContext context = getInterpreterContext();
-    InterpreterResult result = interpreter.interpret("%matplotlib inline\nimport matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context);
+    InterpreterResult result = interpreter.interpret("%matplotlib inline\n" +
+        "import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
+    List<InterpreterResultMessage> interpreterResultMessages =
+        context.out.toInterpreterResultMessage();
     // the order of IMAGE and TEXT is not determined
     // check there must be one IMAGE output
     boolean hasImageOutput = false;
@@ -202,13 +204,15 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
     startInterpreter(properties);
 
     // to make this test can run under both python2 and python3
-    InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext());
+    InterpreterResult result =
+        interpreter.interpret("from __future__ import print_function", getInterpreterContext());
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
 
     InterpreterContext context = getInterpreterContext();
     result = interpreter.interpret("print('1'*3000)", context);
     assertEquals(InterpreterResult.Code.ERROR, result.code());
-    List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
+    List<InterpreterResultMessage> interpreterResultMessages =
+        context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
     assertTrue(interpreterResultMessages.get(0).getData().contains("exceeds maximum: 3000"));
 
@@ -222,7 +226,8 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
     properties.setProperty("zeppelin.ipython.grpc.message_size", "5000");
     startInterpreter(properties);
     // to make this test can run under both python2 and python3
-    result = interpreter.interpret("from __future__ import print_function", getInterpreterContext());
+    result =
+        interpreter.interpret("from __future__ import print_function", getInterpreterContext());
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
 
     context = getInterpreterContext();

python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java

@@ -18,19 +18,30 @@
 package org.apache.zeppelin.python;
 
 
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.regex.Matcher;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class PythonCondaInterpreterTest {
   private PythonCondaInterpreter conda;
@@ -124,20 +135,9 @@ public class PythonCondaInterpreterTest {
   }
 
   private InterpreterContext getInterpreterContext() {
-    return new InterpreterContext(
-        "noteId",
-        "paragraphId",
-        null,
-        "paragraphTitle",
-        "paragraphText",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        null,
-        null,
-        null,
-        new InterpreterOutput(null));
+    return InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .build();
   }
 
 

python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java

@@ -1,34 +1,31 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.zeppelin.python;
 
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 
 import java.io.File;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.Properties;
 
 import static org.mockito.Matchers.any;
@@ -80,19 +77,8 @@ public class PythonDockerInterpreterTest {
   }
 
   private InterpreterContext getInterpreterContext() {
-    return new InterpreterContext(
-        "noteId",
-        "paragraphId",
-        "replName",
-        "paragraphTitle",
-        "paragraphText",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        null,
-        null,
-        null,
-        new InterpreterOutput(null));
+    return InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .build();
   }
 }

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java

@@ -17,30 +17,26 @@
 
 package org.apache.zeppelin.python;
 
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
 
 public class PythonInterpreterMatplotlibTest implements InterpreterOutputListener {
   private InterpreterGroup intpGroup;
@@ -67,15 +63,9 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
 
     out = new InterpreterOutput(this);
 
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        out);
+    context = InterpreterContext.builder()
+        .setInterpreterOut(out)
+        .build();
     python.open();
   }
 
@@ -104,9 +94,12 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
     ret = python.interpret("plt.plot([1, 2, 3])", context);
     ret = python.interpret("plt.show()", context);
 
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Type.TEXT, out.getOutputAt(0).getType());
-    assertEquals(new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Type.HTML, out.getOutputAt(1).getType());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()),
+        InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()),
+        InterpreterResult.Type.TEXT, out.getOutputAt(0).getType());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()),
+        InterpreterResult.Type.HTML, out.getOutputAt(1).getType());
     assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("data:image/png;base64"));
     assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("<div>"));
   }
@@ -128,17 +121,18 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
     // type to HTML.
     ret = python.interpret("plt.show()", context);
 
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()),
+        InterpreterResult.Code.SUCCESS, ret.code());
     assertEquals(0, ret.message().size());
 
     // Now test that new plot is drawn. It should be identical to the
     // previous one.
     ret = python.interpret("plt.plot([1, 2, 3])", context);
-    String msg1 =  new String(out.getOutputAt(0).toByteArray());
+    String msg1 = new String(out.getOutputAt(0).toByteArray());
     InterpreterResult.Type type1 = out.getOutputAt(0).getType();
 
     ret2 = python.interpret("plt.show()", context);
-    String msg2 =  new String(out.getOutputAt(0).toByteArray());
+    String msg2 = new String(out.getOutputAt(0).toByteArray());
     InterpreterResult.Type type2 = out.getOutputAt(0).getType();
 
     assertEquals(msg1, msg2);
@@ -161,16 +155,16 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
     // of FigureManager, causing show() to set the output
     // type to HTML even though the figure is inactive.
     ret = python.interpret("plt.show()", context);
-    String msg1 =  new String(out.getOutputAt(0).toByteArray());
+    String msg1 = new String(out.getOutputAt(0).toByteArray());
     assertNotSame("", msg1);
 
     // Now test that plot can be reshown if it is updated. It should be
     // different from the previous one because it will plot the same line
     // again but in a different color.
     ret = python.interpret("plt.plot([1, 2, 3])", context);
-    msg1 =  new String(out.getOutputAt(1).toByteArray());
+    msg1 = new String(out.getOutputAt(1).toByteArray());
     ret2 = python.interpret("plt.show()", context);
-    String msg2 =  new String(out.getOutputAt(1).toByteArray());
+    String msg2 = new String(out.getOutputAt(1).toByteArray());
 
     assertNotSame(msg1, msg2);
   }

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java

@@ -1,38 +1,23 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
@@ -40,24 +25,30 @@ import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 /**
  * In order for this test to work, test env must have installed:
  * <ol>
- *  - <li>Python</li>
- *  - <li>NumPy</li>
- *  - <li>Pandas</li>
- *  - <li>PandaSql</li>
+ * - <li>Python</li>
+ * - <li>NumPy</li>
+ * - <li>Pandas</li>
+ * - <li>PandaSql</li>
  * <ol>
- *
+ * <p>
  * To run manually on such environment, use:
  * <code>
- *   mvn -Dpython.test.exclude='' test -pl python -am
+ * mvn -Dpython.test.exclude='' test -pl python -am
  * </code>
  */
 public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener {
@@ -79,15 +70,9 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     intpGroup = new InterpreterGroup();
 
     out = new InterpreterOutput(this);
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        out);
+    context = InterpreterContext.builder()
+        .setInterpreterOut(out)
+        .build();
     InterpreterContext.set(context);
 
     python = new PythonInterpreter(p);
@@ -100,7 +85,6 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     intpGroup.put("note", Arrays.asList(python, sql));
 
 
-
     // to make sure python is running.
     InterpreterResult ret = python.interpret("\n", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
@@ -115,7 +99,8 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
 
   @Test
   public void dependenciesAreInstalled() throws InterpreterException {
-    InterpreterResult ret = python.interpret("import pandas\nimport pandasql\nimport numpy\n", context);
+    InterpreterResult ret =
+        python.interpret("import pandas\nimport pandasql\nimport numpy\n", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
   }
 
@@ -137,7 +122,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     ret = python.interpret("import pandas as pd", context);
     ret = python.interpret("import numpy as np", context);
     // DataFrame df2 \w test data
-    ret = python.interpret("df2 = pd.DataFrame({ 'age'  : np.array([33, 51, 51, 34]), "+
+    ret = python.interpret("df2 = pd.DataFrame({ 'age'  : np.array([33, 51, 51, 34]), " +
         "'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
     assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
 
@@ -145,12 +130,16 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     ret = sql.interpret("select name, age from df2 where age < 40", context);
 
     //then
-    assertEquals(new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()),
+        InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE,
+        out.getOutputAt(1).getType());
     assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("moon\t33") > 0);
     assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("park\t34") > 0);
 
-    assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from df2", context).code());
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        sql.interpret("select case when name==\"aa\" then name else name end from df2",
+            context).code());
   }
 
   @Test
@@ -179,8 +168,10 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
     ret = python.interpret("z.show(df1, show_index=True)", context);
 
     // then
-    assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
-    assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType());
+    assertEquals(new String(out.getOutputAt(0).toByteArray()),
+        InterpreterResult.Code.SUCCESS, ret.code());
+    assertEquals(new String(out.getOutputAt(1).toByteArray()),
+        Type.TABLE, out.getOutputAt(1).getType());
     assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("index_name"));
     assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("nan"));
     assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("6.7"));
@@ -200,4 +191,4 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
   public void onUpdate(int index, InterpreterResultMessageOutput out) {
 
   }
-}
+}

python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java

@@ -36,6 +36,7 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+
 public class PythonInterpreterTest extends BasePythonInterpreterTest {
 
   @Override
@@ -78,7 +79,7 @@ public class PythonInterpreterTest extends BasePythonInterpreterTest {
   private class infinityPythonJob implements Runnable {
     @Override
     public void run() {
-      String code = "import time\nwhile True:\n  time.sleep(1)" ;
+      String code = "import time\nwhile True:\n  time.sleep(1)";
       InterpreterResult ret = null;
       try {
         ret = interpreter.interpret(code, getInterpreterContext());

scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java

@@ -17,9 +17,10 @@
 
 package org.apache.zeppelin.scalding;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.FixMethodOrder;
@@ -27,18 +28,10 @@ import org.junit.Test;
 import org.junit.runners.MethodSorters;
 
 import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
 import java.util.Properties;
 
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
-import org.apache.zeppelin.user.AuthenticationInfo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests for the Scalding interpreter for Zeppelin.
@@ -66,11 +59,11 @@ public class ScaldingInterpreterTest {
       repl.open();
     }
 
-    InterpreterGroup intpGroup = new InterpreterGroup();
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(), new GUI(), new AngularObjectRegistry(
-            intpGroup.getId(), null), null,
-        new LinkedList<InterpreterContextRunner>(), null);
+    context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setAuthenticationInfo(new AuthenticationInfo())
+        .build();
   }
 
   @After

scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java

@@ -40,15 +40,10 @@ public class ScioInterpreterTest {
   private final String newline = "\n";
 
   private InterpreterContext getNewContext() {
-    return new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        new InterpreterOutput(null));
+    return InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .build();
   }
 
   @Before

shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java

@@ -42,8 +42,7 @@ public class ShellInterpreterTest {
     p.setProperty("shell.command.timeout.millisecs", "2000");
     shell = new ShellInterpreter(p);
 
-    context = new InterpreterContext("", "1", null, "", "", null, null, null, null, null, null,
-        null, null);
+    context = InterpreterContext.builder().setParagraphId("paragraphId").build();
     shell.open();
   }
 

spark/interpreter/pom.xml

@@ -131,6 +131,18 @@
       <artifactId>spark-core_${scala.binary.version}</artifactId>
       <version>${spark.version}</version>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-client</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>2.6.0</version>
+      <scope>provided</scope>
     </dependency>
 
     <dependency>

spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkInterpreter.java

@@ -31,6 +31,8 @@ import java.util.Properties;
  */
 public abstract class AbstractSparkInterpreter extends Interpreter {
 
+  private SparkInterpreter parentSparkInterpreter;
+
   public AbstractSparkInterpreter(Properties properties) {
     super(properties);
   }
@@ -54,4 +56,12 @@ public abstract class AbstractSparkInterpreter extends Interpreter {
   public abstract String getSparkUIUrl();
 
   public abstract boolean isUnsupportedSparkVersion();
+
+  public void setParentSparkInterpreter(SparkInterpreter parentSparkInterpreter) {
+    this.parentSparkInterpreter = parentSparkInterpreter;
+  }
+
+  public SparkInterpreter getParentSparkInterpreter() {
+    return parentSparkInterpreter;
+  }
 }

spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java

@@ -117,7 +117,11 @@ public class IPySparkInterpreter extends IPythonInterpreter {
     String jobGroupId = Utils.buildJobGroupId(context);
     String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo());
     String setJobGroupStmt = "sc.setJobGroup('" +  jobGroupId + "', '" + jobDesc + "')";
-    return super.interpret(setJobGroupStmt +"\n" + st, context);
+    InterpreterResult result = super.interpret(setJobGroupStmt, context);
+    if (result.code().equals(InterpreterResult.Code.ERROR)) {
+      return new InterpreterResult(InterpreterResult.Code.ERROR, "Fail to setJobGroup");
+    }
+    return super.interpret(st, context);
   }
 
   @Override

spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java

@@ -23,10 +23,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.SparkContext;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.scheduler.SparkListenerJobStart;
 import org.apache.spark.sql.SQLContext;
-import org.apache.spark.ui.jobs.JobProgressListener;
-import org.apache.zeppelin.interpreter.BaseZeppelinContext;
 import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -34,7 +31,6 @@ import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.spark.dep.SparkDependencyContext;
 import org.slf4j.Logger;
@@ -42,8 +38,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -87,7 +81,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
     try {
       String scalaVersion = extractScalaVersion();
       LOGGER.info("Using Scala Version: " + scalaVersion);
-      setupConfForPySpark();
       SparkConf conf = new SparkConf();
       for (Map.Entry<Object, Object> entry : getProperties().entrySet()) {
         if (!StringUtils.isBlank(entry.getValue().toString())) {
@@ -102,9 +95,10 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
 
       String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion);
       Class clazz = Class.forName(innerIntpClassName);
-      this.innerInterpreter =
-          (BaseSparkScalaInterpreter) clazz.getConstructor(SparkConf.class, List.class)
-              .newInstance(conf, getDependencyFiles());
+      this.innerInterpreter = (BaseSparkScalaInterpreter)
+          clazz.getConstructor(SparkConf.class, List.class, Boolean.class)
+              .newInstance(conf, getDependencyFiles(),
+                  Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput", "true")));
       this.innerInterpreter.open();
 
       sc = this.innerInterpreter.sc();
@@ -117,63 +111,21 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
       }
       sqlContext = this.innerInterpreter.sqlContext();
       sparkSession = this.innerInterpreter.sparkSession();
-      sparkUrl = this.innerInterpreter.sparkUrl();
-      sparkShims = SparkShims.getInstance(sc.version());
-      sparkShims.setupSparkListener(sparkUrl);
-
       hooks = getInterpreterGroup().getInterpreterHookRegistry();
       z = new SparkZeppelinContext(sc, hooks,
           Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
       this.innerInterpreter.bind("z", z.getClass().getCanonicalName(), z,
           Lists.newArrayList("@transient"));
+
+      sparkUrl = this.innerInterpreter.sparkUrl();
+      sparkShims = SparkShims.getInstance(sc.version());
+      sparkShims.setupSparkListener(sc.master(), sparkUrl, InterpreterContext.get());
     } catch (Exception e) {
       LOGGER.error("Fail to open SparkInterpreter", ExceptionUtils.getStackTrace(e));
       throw new InterpreterException("Fail to open SparkInterpreter", e);
     }
   }
 
-  private void setupConfForPySpark() {
-    String sparkHome = getProperty("SPARK_HOME");
-    File pysparkFolder = null;
-    if (sparkHome == null) {
-      String zeppelinHome =
-          new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../../")
-              .getValue().toString();
-      pysparkFolder = new File(zeppelinHome,
-          "interpreter" + File.separator + "spark" + File.separator + "pyspark");
-    } else {
-      pysparkFolder = new File(sparkHome, "python" + File.separator + "lib");
-    }
-
-    ArrayList<String> pysparkPackages = new ArrayList<>();
-    for (File file : pysparkFolder.listFiles()) {
-      if (file.getName().equals("pyspark.zip")) {
-        pysparkPackages.add(file.getAbsolutePath());
-      }
-      if (file.getName().startsWith("py4j-")) {
-        pysparkPackages.add(file.getAbsolutePath());
-      }
-    }
-
-    if (pysparkPackages.size() != 2) {
-      throw new RuntimeException("Not correct number of pyspark packages: " +
-          StringUtils.join(pysparkPackages, ","));
-    }
-    // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
-    System.setProperty("spark.files", mergeProperty(System.getProperty("spark.files", ""),
-        StringUtils.join(pysparkPackages, ",")));
-    System.setProperty("spark.submit.pyFiles", mergeProperty(
-        System.getProperty("spark.submit.pyFiles", ""), StringUtils.join(pysparkPackages, ",")));
-
-  }
-
-  private String mergeProperty(String originalValue, String appendedValue) {
-    if (StringUtils.isBlank(originalValue)) {
-      return appendedValue;
-    }
-    return originalValue + "," + appendedValue;
-  }
-
   @Override
   public void close() {
     LOGGER.info("Close SparkInterpreter");
@@ -244,7 +196,8 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
   }
 
   private DepInterpreter getDepInterpreter() {
-    Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
+    Interpreter p = getParentSparkInterpreter()
+        .getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
     if (p == null) {
       return null;
     }
@@ -279,10 +232,9 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
         infos.put("message", "No spark url defined");
       }
     }
-    if (ctx != null && ctx.getClient() != null) {
+    if (ctx != null) {
       LOGGER.debug("Sending metadata to Zeppelin server: {}", infos.toString());
-      getZeppelinContext().setEventClient(ctx.getClient());
-      ctx.getClient().onMetaInfosReceived(infos);
+      ctx.getIntpEventClient().onMetaInfosReceived(infos);
     }
   }
 

spark/interpreter/src/main/java/org/apache/zeppelin/spark/OldSparkInterpreter.java

@@ -162,8 +162,6 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter {
     this(property);
     this.sc = sc;
     env = SparkEnv.get();
-    sparkShims = SparkShims.getInstance(sc.version());
-    sparkShims.setupSparkListener(sparkUrl);
   }
 
   public SparkContext getSparkContext() {
@@ -281,7 +279,8 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter {
   }
 
   private DepInterpreter getDepInterpreter() {
-    Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
+    Interpreter p = getParentSparkInterpreter()
+        .getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
     if (p == null) {
       return null;
     }
@@ -872,8 +871,7 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter {
 
     sparkUrl = getSparkUIUrl();
     sparkShims = SparkShims.getInstance(sc.version());
-    sparkShims.setupSparkListener(sparkUrl);
-
+    sparkShims.setupSparkListener(sc.master(), sparkUrl, InterpreterContext.get());
     numReferenceOfSparkContext.incrementAndGet();
   }
 
@@ -926,10 +924,9 @@ public class OldSparkInterpreter extends AbstractSparkInterpreter {
         infos.put("message", "No spark url defined");
       }
     }
-    if (ctx != null && ctx.getClient() != null) {
+    if (ctx != null) {
       logger.info("Sending metadata to Zeppelin server: {}", infos.toString());
-      getZeppelinContext().setEventClient(ctx.getClient());
-      ctx.getClient().onMetaInfosReceived(infos);
+      ctx.getIntpEventClient().onMetaInfosReceived(infos);
     }
   }
 

spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -130,11 +130,11 @@ public class PySparkInterpreter extends PythonInterpreter {
     try {
       URLClassLoader newCl = new URLClassLoader(urls, oldCl);
       Thread.currentThread().setContextClassLoader(newCl);
-      // create Python Process and JVM gateway
-      super.open();
       // must create spark interpreter after ClassLoader is set, otherwise the additional jars
       // can not be loaded by spark repl.
       this.sparkInterpreter = getSparkInterpreter();
+      // create Python Process and JVM gateway
+      super.open();
     } finally {
       Thread.currentThread().setContextClassLoader(oldCl);
     }
@@ -175,22 +175,27 @@ public class PySparkInterpreter extends PythonInterpreter {
     String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo());
     callPython(new PythonInterpretRequest(
         String.format("if 'sc' in locals():\n\tsc.setJobGroup('%s', '%s')", jobGroup, jobDesc),
-        false));
+        false, false));
   }
 
   // Run python shell
   // Choose python in the order of
-  // PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
+  // spark.pyspark.driver.python > spark.pyspark.python > PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON
   @Override
   protected String getPythonExec() {
-    String pythonExec = getProperty("zeppelin.pyspark.python", "python");
+    if (!StringUtils.isBlank(getProperty("spark.pyspark.driver.python", ""))) {
+      return properties.getProperty("spark.pyspark.driver.python");
+    }
+    if (!StringUtils.isBlank(getProperty("spark.pyspark.python", ""))) {
+      return properties.getProperty("spark.pyspark.python");
+    }
     if (System.getenv("PYSPARK_PYTHON") != null) {
-      pythonExec = System.getenv("PYSPARK_PYTHON");
+      return System.getenv("PYSPARK_PYTHON");
     }
     if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
-      pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
+      return System.getenv("PYSPARK_DRIVER_PYTHON");
     }
-    return pythonExec;
+    return "python";
   }
 
   @Override

spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -50,6 +50,7 @@ public class SparkInterpreter extends AbstractSparkInterpreter {
     } else {
       delegation = new OldSparkInterpreter(properties);
     }
+    delegation.setParentSparkInterpreter(this);
   }
 
   @Override

spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java

@@ -33,6 +33,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -53,6 +54,7 @@ public class SparkRInterpreter extends Interpreter {
   private AtomicBoolean rbackendDead = new AtomicBoolean(false);
   private SparkContext sc;
   private JavaSparkContext jsc;
+  private String secret;
 
   public SparkRInterpreter(Properties property) {
     super(property);
@@ -75,20 +77,21 @@ public class SparkRInterpreter extends Interpreter {
       // yarn-cluster mode
       sparkRLibPath = "sparkr";
     }
-
-    // Share the same SparkRBackend across sessions
-    synchronized (SparkRBackend.backend()) {
-      if (!SparkRBackend.isStarted()) {
-        SparkRBackend.init();
-        SparkRBackend.start();
-      }
+    if (!new File(sparkRLibPath).exists()) {
+      throw new InterpreterException(String.format("sparkRLib %s doesn't exist", sparkRLibPath));
     }
 
-    int port = SparkRBackend.port();
     this.sparkInterpreter = getSparkInterpreter();
     this.sc = sparkInterpreter.getSparkContext();
     this.jsc = sparkInterpreter.getJavaSparkContext();
+    // Share the same SparkRBackend across sessions
     SparkVersion sparkVersion = new SparkVersion(sc.version());
+    synchronized (SparkRBackend.backend()) {
+      if (!SparkRBackend.isStarted()) {
+        SparkRBackend.init(sparkVersion);
+        SparkRBackend.start();
+      }
+    }
     this.isSpark2 = sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0);
     int timeout = this.sc.getConf().getInt("spark.r.backendConnectionTimeout", 6000);
 
@@ -100,7 +103,7 @@ public class SparkRInterpreter extends Interpreter {
     ZeppelinRContext.setSqlContext(sparkInterpreter.getSQLContext());
     ZeppelinRContext.setZeppelinContext(sparkInterpreter.getZeppelinContext());
 
-    zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion, timeout, this);
+    zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, SparkRBackend.port(), sparkVersion, timeout, this);
     try {
       zeppelinR.open();
     } catch (IOException e) {

spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java

@@ -34,6 +34,7 @@ public class SparkVersion {
   public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0");
 
   public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0");
+  public static final SparkVersion SPARK_2_3_1 = SparkVersion.fromVersionString("2.3.1");
   public static final SparkVersion SPARK_2_4_0 = SparkVersion.fromVersionString("2.4.0");
 
   public static final SparkVersion MIN_SUPPORTED_VERSION =  SPARK_1_0_0;
@@ -108,6 +109,9 @@ public class SparkVersion {
     return this.olderThan(SPARK_1_3_0);
   }
 
+  public boolean isSecretSocketSupported() {
+    return this.newerThanEquals(SPARK_2_3_1);
+  }
   public boolean equals(Object versionToCompare) {
     return version == ((SparkVersion) versionToCompare).version;
   }

spark/interpreter/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java

@@ -19,6 +19,7 @@ package org.apache.zeppelin.spark;
 import org.apache.commons.exec.*;
 import org.apache.commons.exec.environment.EnvironmentUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkRBackend;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@@ -146,7 +147,9 @@ public class ZeppelinR implements ExecuteResultHandler {
     cmd.addArgument(libPath);
     cmd.addArgument(Integer.toString(sparkVersion.toNumber()));
     cmd.addArgument(Integer.toString(timeout));
-    
+    if (sparkVersion.isSecretSocketSupported()) {
+      cmd.addArgument(SparkRBackend.socketSecret());
+    }
     // dump out the R command to facilitate manually running it, e.g. for fault diagnosis purposes
     logger.debug(cmd.toString());
 

spark/interpreter/src/main/resources/R/zeppelin_sparkr.R

@@ -23,6 +23,11 @@ port <- as.integer(args[2])
 libPath <- args[3]
 version <- as.integer(args[4])
 timeout <- as.integer(args[5])
+authSecret <- NULL
+if (length(args) >= 6) {
+  authSecret <- args[6]
+}
+
 rm(args)
 
 print(paste("Port ", toString(port)))
@@ -31,8 +36,11 @@ print(paste("LibPath ", libPath))
 .libPaths(c(file.path(libPath), .libPaths()))
 library(SparkR)
 
-
-SparkR:::connectBackend("localhost", port, timeout)
+if (is.null(authSecret)) {
+  SparkR:::connectBackend("localhost", port, timeout)
+} else {
+  SparkR:::connectBackend("localhost", port, timeout, authSecret)
+}
 
 # scStartTime is needed by R/pkg/R/sparkR.R
 assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv)

spark/interpreter/src/main/resources/interpreter-setting.json

@@ -78,7 +78,7 @@
       "zeppelin.spark.useNew": {
         "envName": null,
         "propertyName": "zeppelin.spark.useNew",
-        "defaultValue": "false",
+        "defaultValue": true,
         "description": "Whether use new spark interpreter implementation",
         "type": "checkbox"
       }
@@ -170,9 +170,16 @@
     "name": "pyspark",
     "className": "org.apache.zeppelin.spark.PySparkInterpreter",
     "properties": {
-      "zeppelin.pyspark.python": {
+      "PYSPARK_PYTHON": {
         "envName": "PYSPARK_PYTHON",
-        "propertyName": null,
+        "propertyName": "PYSPARK_PYTHON",
+        "defaultValue": "python",
+        "description": "Python command to run pyspark with",
+        "type": "string"
+      },
+      "PYSPARK_DRIVER_PYTHON": {
+        "envName": "PYSPARK_DRIVER_PYTHON",
+        "propertyName": "PYSPARK_DRIVER_PYTHON",
         "defaultValue": "python",
         "description": "Python command to run pyspark with",
         "type": "string"

spark/interpreter/src/main/scala/org/apache/spark/SparkRBackend.scala

@@ -17,11 +17,13 @@
 package org.apache.spark
 
 import org.apache.spark.api.r.RBackend
+import org.apache.zeppelin.spark.SparkVersion
 
 object SparkRBackend {
   val backend : RBackend = new RBackend()
   private var started = false;
   private var portNumber = 0;
+  private var secret: String = "";
 
   val backendThread : Thread = new Thread("SparkRBackend") {
     override def run() {
@@ -29,9 +31,16 @@ object SparkRBackend {
     }
   }
 
-  def init() : Int = {
-    portNumber = backend.init()
-    portNumber
+  def init(version: SparkVersion) : Unit = {
+    val rBackendClass = classOf[RBackend]
+    if (version.isSecretSocketSupported) {
+      val result = rBackendClass.getMethod("init").invoke(backend).asInstanceOf[Tuple2[Int, Object]]
+      portNumber = result._1
+      val rAuthHelper = result._2
+      secret = rAuthHelper.getClass.getMethod("secret").invoke(rAuthHelper).asInstanceOf[String]
+    } else {
+      portNumber = rBackendClass.getMethod("init").invoke(backend).asInstanceOf[Int]
+    }
   }
 
   def start() : Unit = {
@@ -44,11 +53,9 @@ object SparkRBackend {
     backendThread.join()
   }
 
-  def isStarted() : Boolean = {
-    started
-  }
+  def isStarted() : Boolean = started
 
-  def port(): Int = {
-    return portNumber
-  }
+  def port(): Int = portNumber
+
+  def socketSecret(): String = secret;
 }

spark/interpreter/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java

@@ -17,17 +17,10 @@
 
 package org.apache.zeppelin.spark;
 
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Properties;
-
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.junit.After;
 import org.junit.Before;
@@ -35,6 +28,12 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
 public class DepInterpreterTest {
 
   @Rule
@@ -63,11 +62,8 @@ public class DepInterpreterTest {
     intpGroup.get("note").add(dep);
     dep.setInterpreterGroup(intpGroup);
 
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(), new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        null,
-        new LinkedList<InterpreterContextRunner>(), null);
+    context = InterpreterContext.builder()
+        .build();;
   }
 
   @After

spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java

@@ -19,7 +19,6 @@ package org.apache.zeppelin.spark;
 
 
 import com.google.common.io.Files;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
@@ -28,15 +27,13 @@ import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.python.IPythonInterpreterTest;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -51,7 +48,7 @@ import static org.mockito.Mockito.verify;
 public class IPySparkInterpreterTest extends IPythonInterpreterTest {
 
   private InterpreterGroup intpGroup;
-  private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class);
+  private RemoteInterpreterEventClient mockIntpEventClient = mock(RemoteInterpreterEventClient.class);
 
   @Override
   protected Properties initIntpProperties() {
@@ -70,13 +67,17 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
     return p;
   }
 
+
   @Override
   protected void startInterpreter(Properties properties) throws InterpreterException {
-    intpGroup = new InterpreterGroup();
-    intpGroup.put("session_1", new ArrayList<Interpreter>());
+    InterpreterContext context = getInterpreterContext();
+    context.setIntpEventClient(mockIntpEventClient);
+    InterpreterContext.set(context);
 
     LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
         new SparkInterpreter(properties));
+    intpGroup = new InterpreterGroup();
+    intpGroup.put("session_1", new ArrayList<Interpreter>());
     intpGroup.get("session_1").add(sparkInterpreter);
     sparkInterpreter.setInterpreterGroup(intpGroup);
 
@@ -102,32 +103,32 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
 
   @Test
   public void testIPySpark() throws InterruptedException, InterpreterException, IOException {
-    testPySpark(interpreter, mockRemoteEventClient);
+    testPySpark(interpreter, mockIntpEventClient);
   }
 
-  public static void testPySpark(final Interpreter interpreter, RemoteEventClient mockRemoteEventClient)
+  public static void testPySpark(final Interpreter interpreter, RemoteInterpreterEventClient mockIntpEventClient)
       throws InterpreterException, IOException, InterruptedException {
-    reset(mockRemoteEventClient);
+    reset(mockIntpEventClient);
     // rdd
-    InterpreterContext context = createInterpreterContext(mockRemoteEventClient);
+    InterpreterContext context = createInterpreterContext(mockIntpEventClient);
     InterpreterResult result = interpreter.interpret("sc.version", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     String sparkVersion = context.out.toInterpreterResultMessage().get(0).getData();
     // spark url is sent
-    verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class));
+    verify(mockIntpEventClient).onMetaInfosReceived(any(Map.class));
 
-    context = createInterpreterContext(mockRemoteEventClient);
+    context = createInterpreterContext(mockIntpEventClient);
     result = interpreter.interpret("sc.range(1,10).sum()", context);
     Thread.sleep(100);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals("45", interpreterResultMessages.get(0).getData().trim());
     // spark job url is sent
-//    verify(mockRemoteEventClient).onParaInfosReceived(any(String.class), any(String.class), any(Map.class));
+    verify(mockIntpEventClient).onParaInfosReceived(any(Map.class));
 
     // spark sql
-    context = createInterpreterContext(mockRemoteEventClient);
+    context = createInterpreterContext(mockIntpEventClient);
     if (!isSpark2(sparkVersion)) {
       result = interpreter.interpret("df = sqlContext.createDataFrame([(1,'a'),(2,'b')])\ndf.show()", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -140,7 +141,7 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
               "|  2|  b|\n" +
               "+---+---+", interpreterResultMessages.get(0).getData().trim());
 
-      context = createInterpreterContext(mockRemoteEventClient);
+      context = createInterpreterContext(mockIntpEventClient);
       result = interpreter.interpret("z.show(df)", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       interpreterResultMessages = context.out.toInterpreterResultMessage();
@@ -160,7 +161,7 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
               "|  2|  b|\n" +
               "+---+---+", interpreterResultMessages.get(0).getData().trim());
 
-      context = createInterpreterContext(mockRemoteEventClient);
+      context = createInterpreterContext(mockIntpEventClient);
       result = interpreter.interpret("z.show(df)", context);
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       interpreterResultMessages = context.out.toInterpreterResultMessage();
@@ -171,7 +172,7 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
     }
     // cancel
     if (interpreter instanceof IPySparkInterpreter) {
-      final InterpreterContext context2 = createInterpreterContext(mockRemoteEventClient);
+      final InterpreterContext context2 = createInterpreterContext(mockIntpEventClient);
 
       Thread thread = new Thread() {
         @Override
@@ -201,24 +202,24 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
     }
 
     // completions
-    List<InterpreterCompletion> completions = interpreter.completion("sc.ran", 6, createInterpreterContext(mockRemoteEventClient));
+    List<InterpreterCompletion> completions = interpreter.completion("sc.ran", 6, createInterpreterContext(mockIntpEventClient));
     assertEquals(1, completions.size());
     assertEquals("range", completions.get(0).getValue());
 
-    completions = interpreter.completion("sc.", 3, createInterpreterContext(mockRemoteEventClient));
+    completions = interpreter.completion("sc.", 3, createInterpreterContext(mockIntpEventClient));
     assertTrue(completions.size() > 0);
     completions.contains(new InterpreterCompletion("range", "range", ""));
 
-    completions = interpreter.completion("1+1\nsc.", 7, createInterpreterContext(mockRemoteEventClient));
+    completions = interpreter.completion("1+1\nsc.", 7, createInterpreterContext(mockIntpEventClient));
     assertTrue(completions.size() > 0);
     completions.contains(new InterpreterCompletion("range", "range", ""));
 
-    completions = interpreter.completion("s", 1, createInterpreterContext(mockRemoteEventClient));
+    completions = interpreter.completion("s", 1, createInterpreterContext(mockIntpEventClient));
     assertTrue(completions.size() > 0);
     completions.contains(new InterpreterCompletion("sc", "sc", ""));
 
     // pyspark streaming
-    context = createInterpreterContext(mockRemoteEventClient);
+    context = createInterpreterContext(mockIntpEventClient);
     result = interpreter.interpret(
         "from pyspark.streaming import StreamingContext\n" +
             "import time\n" +
@@ -244,22 +245,12 @@ public class IPySparkInterpreterTest extends IPythonInterpreterTest {
     return sparkVersion.startsWith("'2.") || sparkVersion.startsWith("u'2.");
   }
 
-  private static InterpreterContext createInterpreterContext(RemoteEventClient mockRemoteEventClient) {
-    InterpreterContext context = new InterpreterContext(
-        "noteId",
-        "paragraphId",
-        "replName",
-        "paragraphTitle",
-        "paragraphText",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        null,
-        null,
-        null,
-        new InterpreterOutput(null));
-    context.setClient(mockRemoteEventClient);
-    return context;
+  private static InterpreterContext createInterpreterContext(RemoteInterpreterEventClient mockRemoteEventClient) {
+    return InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setIntpEventClient(mockRemoteEventClient)
+        .setInterpreterOut(new InterpreterOutput(null))
+        .build();
   }
 }

spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java

@@ -17,11 +17,12 @@
 
 package org.apache.zeppelin.spark;
 
+import com.google.common.io.Files;
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.display.ui.CheckBox;
 import org.apache.zeppelin.display.ui.Select;
 import org.apache.zeppelin.display.ui.TextBox;
+import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
@@ -29,11 +30,10 @@ import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.File;
@@ -42,6 +42,7 @@ import java.io.IOException;
 import java.net.URL;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -57,13 +58,14 @@ import static org.mockito.Mockito.verify;
 public class NewSparkInterpreterTest {
 
   private SparkInterpreter interpreter;
+  private DepInterpreter depInterpreter;
 
   // catch the streaming output in onAppend
   private volatile String output = "";
   // catch the interpreter output in onUpdate
   private InterpreterResultMessageOutput messageOutput;
 
-  private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class);
+  private RemoteInterpreterEventClient mockRemoteEventClient = mock(RemoteInterpreterEventClient.class);
 
   @Test
   public void testSparkInterpreter() throws IOException, InterruptedException, InterpreterException {
@@ -73,12 +75,19 @@ public class NewSparkInterpreterTest {
     properties.setProperty("zeppelin.spark.maxResult", "100");
     properties.setProperty("zeppelin.spark.test", "true");
     properties.setProperty("zeppelin.spark.useNew", "true");
+
+    InterpreterContext context = InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mockRemoteEventClient)
+        .setAngularObjectRegistry(new AngularObjectRegistry("spark", null))
+        .build();
+    InterpreterContext.set(context);
+
     interpreter = new SparkInterpreter(properties);
     assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
     interpreter.setInterpreterGroup(mock(InterpreterGroup.class));
     interpreter.open();
 
-    interpreter.getZeppelinContext().setEventClient(mockRemoteEventClient);
     InterpreterResult result = interpreter.interpret("val a=\"hello world\"", getInterpreterContext());
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertEquals("a: String = hello world\n", output);
@@ -134,7 +143,7 @@ public class NewSparkInterpreterTest {
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertTrue(output.contains("45"));
     // spark job url is sent
-    verify(mockRemoteEventClient).onParaInfosReceived(any(String.class), any(String.class), any(Map.class));
+    verify(mockRemoteEventClient).onParaInfosReceived(any(Map.class));
 
     // case class
     result = interpreter.interpret("val bankText = sc.textFile(\"bank.csv\")", getInterpreterContext());
@@ -199,7 +208,7 @@ public class NewSparkInterpreterTest {
     messageOutput.flush();
     assertEquals("_1\t_2\n1\ta\n2\tb\n", messageOutput.toInterpreterResultMessage().getData());
 
-    InterpreterContext context = getInterpreterContext();
+    context = getInterpreterContext();
     result = interpreter.interpret("z.input(\"name\", \"default_name\")", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertEquals(1, context.getGui().getForms().size());
@@ -351,28 +360,80 @@ public class NewSparkInterpreterTest {
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
   }
 
+  //TODO(zjffdu) This unit test will fail due to classpath issue, should enable it after the classpath issue is fixed.
+  @Ignore
+  public void testDepInterpreter() throws InterpreterException {
+    Properties properties = new Properties();
+    properties.setProperty("spark.master", "local");
+    properties.setProperty("spark.app.name", "test");
+    properties.setProperty("zeppelin.spark.maxResult", "100");
+    properties.setProperty("zeppelin.spark.test", "true");
+    properties.setProperty("zeppelin.spark.useNew", "true");
+    properties.setProperty("zeppelin.dep.localrepo", Files.createTempDir().getAbsolutePath());
+
+    InterpreterGroup intpGroup = new InterpreterGroup();
+    interpreter = new SparkInterpreter(properties);
+    depInterpreter = new DepInterpreter(properties);
+    interpreter.setInterpreterGroup(intpGroup);
+    depInterpreter.setInterpreterGroup(intpGroup);
+    intpGroup.put("session_1", new ArrayList<Interpreter>());
+    intpGroup.get("session_1").add(interpreter);
+    intpGroup.get("session_1").add(depInterpreter);
+
+    depInterpreter.open();
+    InterpreterResult result =
+        depInterpreter.interpret("z.load(\"com.databricks:spark-avro_2.11:3.2.0\")", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+
+    interpreter.open();
+    result = interpreter.interpret("import com.databricks.spark.avro._", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+  }
+
+  @Test
+  public void testDisableReplOutput() throws InterpreterException {
+    Properties properties = new Properties();
+    properties.setProperty("spark.master", "local");
+    properties.setProperty("spark.app.name", "test");
+    properties.setProperty("zeppelin.spark.maxResult", "100");
+    properties.setProperty("zeppelin.spark.test", "true");
+    properties.setProperty("zeppelin.spark.useNew", "true");
+    properties.setProperty("zeppelin.spark.printREPLOutput", "false");
+
+    interpreter = new SparkInterpreter(properties);
+    assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
+    interpreter.setInterpreterGroup(mock(InterpreterGroup.class));
+    interpreter.open();
+
+    InterpreterResult result = interpreter.interpret("val a=\"hello world\"", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    // no output for define new variable
+    assertEquals("", output);
+
+    result = interpreter.interpret("print(a)", getInterpreterContext());
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    // output from print statement will still be displayed
+    assertEquals("hello world", output);
+  }
+
   @After
   public void tearDown() throws InterpreterException {
     if (this.interpreter != null) {
       this.interpreter.close();
     }
+    if (this.depInterpreter != null) {
+      this.depInterpreter.close();
+    }
   }
 
   private InterpreterContext getInterpreterContext() {
     output = "";
-    InterpreterContext context = new InterpreterContext(
-        "noteId",
-        "paragraphId",
-        "replName",
-        "paragraphTitle",
-        "paragraphText",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry("spark", null),
-        null,
-        null,
+    InterpreterContext context = InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mockRemoteEventClient)
+        .setAngularObjectRegistry(new AngularObjectRegistry("spark", null))
+        .build();
+    context.out =
         new InterpreterOutput(
 
             new InterpreterOutputListener() {
@@ -394,9 +455,7 @@ public class NewSparkInterpreterTest {
               public void onUpdate(int index, InterpreterResultMessageOutput out) {
                 messageOutput = out;
               }
-            })
-    );
-    context.setClient(mockRemoteEventClient);
+            });
     return context;
   }
 }

spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkSqlInterpreterTest.java

@@ -17,21 +17,26 @@
 
 package org.apache.zeppelin.spark;
 
-import java.util.HashMap;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
 import java.util.LinkedList;
 import java.util.Properties;
 
-import com.google.common.io.Files;
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.junit.*;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
 
 public class NewSparkSqlInterpreterTest {
 
@@ -62,11 +67,15 @@ public class NewSparkSqlInterpreterTest {
     sparkInterpreter.open();
     sqlInterpreter.open();
 
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(), new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
+    context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setParagraphTitle("title")
+        .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+        .setResourcePool(new LocalResourcePool("id"))
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
+        .build();
   }
 
   @AfterClass

spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java

@@ -18,22 +18,19 @@
 package org.apache.zeppelin.spark;
 
 import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.resource.LocalResourcePool;
 import org.apache.zeppelin.resource.WellKnownResourceName;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.AfterClass;
 import org.junit.Assume;
 import org.junit.BeforeClass;
@@ -55,6 +52,7 @@ import java.util.Properties;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class OldSparkInterpreterTest {
@@ -66,8 +64,6 @@ public class OldSparkInterpreterTest {
   static InterpreterGroup intpGroup;
   static InterpreterContext context;
   static Logger LOGGER = LoggerFactory.getLogger(OldSparkInterpreterTest.class);
-  static Map<String, Map<String, String>> paraIdToInfosMap =
-      new HashMap<>();
 
   /**
    * Get spark version number as a numerical value.
@@ -98,41 +94,22 @@ public class OldSparkInterpreterTest {
   @BeforeClass
   public static void setUp() throws Exception {
     intpGroup = new InterpreterGroup();
+    context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setParagraphTitle("title")
+        .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+        .setResourcePool(new LocalResourcePool("id"))
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
+        .build();
+    InterpreterContext.set(context);
+
     intpGroup.put("note", new LinkedList<Interpreter>());
     repl = new SparkInterpreter(getSparkTestProperties(tmpDir));
     repl.setInterpreterGroup(intpGroup);
     intpGroup.get("note").add(repl);
     repl.open();
-
-    final RemoteEventClientWrapper remoteEventClientWrapper = new RemoteEventClientWrapper() {
-
-      @Override
-      public void onParaInfosReceived(String noteId, String paragraphId,
-          Map<String, String> infos) {
-        if (infos != null) {
-          paraIdToInfosMap.put(paragraphId, infos);
-        }
-      }
-
-      @Override
-      public void onMetaInfosReceived(Map<String, String> infos) {
-      }
-    };
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        new InterpreterOutput(null)) {
-
-        @Override
-        public RemoteEventClientWrapper getClient() {
-          return remoteEventClientWrapper;
-        }
-    };
     // The first para interpretdr will set the Eventclient wrapper
     //SparkInterpreter.interpret(String, InterpreterContext) ->
     //SparkInterpreter.populateSparkWebUrl(InterpreterContext) ->
@@ -336,27 +313,4 @@ public class OldSparkInterpreterTest {
     assertTrue(completions.size() > 0);
   }
 
-  @Test
-  public void testParagraphUrls() throws InterpreterException {
-    String paraId = "test_para_job_url";
-    InterpreterContext intpCtx = new InterpreterContext("note", paraId, null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        new InterpreterOutput(null));
-    repl.interpret("sc.parallelize(1 to 10).map(x => {x}).collect", intpCtx);
-    Map<String, String> paraInfos = paraIdToInfosMap.get(intpCtx.getParagraphId());
-    String jobUrl = null;
-    if (paraInfos != null) {
-      jobUrl = paraInfos.get("jobUrl");
-    }
-    String sparkUIUrl = repl.getSparkUIUrl();
-    assertNotNull(jobUrl);
-    assertTrue(jobUrl.startsWith(sparkUIUrl + "/jobs/job?id="));
-
-  }
 }

spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkSqlInterpreterTest.java

@@ -18,29 +18,27 @@
 package org.apache.zeppelin.spark;
 
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.Properties;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
 
 public class OldSparkSqlInterpreterTest {
 
@@ -76,11 +74,15 @@ public class OldSparkSqlInterpreterTest {
     sql.setInterpreterGroup(intpGroup);
     sql.open();
 
-    context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-        new HashMap<String, Object>(), new GUI(), new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
+    context = InterpreterContext.builder()
+        .setNoteId("noteId")
+        .setParagraphId("paragraphId")
+        .setParagraphTitle("title")
+        .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+        .setResourcePool(new LocalResourcePool("id"))
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
+        .build();
   }
 
   @AfterClass

spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java

@@ -18,23 +18,34 @@
 package org.apache.zeppelin.spark;
 
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Type;
-import org.apache.zeppelin.resource.LocalResourcePool;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.junit.*;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.junit.runners.MethodSorters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class PySparkInterpreterMatplotlibTest {
@@ -111,15 +122,12 @@ public class PySparkInterpreterMatplotlibTest {
   public static void setUp() throws Exception {
     intpGroup = new InterpreterGroup();
     intpGroup.put("note", new LinkedList<Interpreter>());
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        new InterpreterOutput(null));
+    context = InterpreterContext.builder()
+        .setNoteId("note")
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mock(RemoteInterpreterEventClient.class))
+        .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+        .build();
     InterpreterContext.set(context);
 
     sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
@@ -131,16 +139,6 @@ public class PySparkInterpreterMatplotlibTest {
     intpGroup.get("note").add(pyspark);
     pyspark.setInterpreterGroup(intpGroup);
     pyspark.open();
-
-    context = new InterpreterContext("note", "id", null, "title", "text",
-        new AuthenticationInfo(),
-        new HashMap<String, Object>(),
-        new GUI(),
-        new GUI(),
-        new AngularObjectRegistry(intpGroup.getId(), null),
-        new LocalResourcePool("id"),
-        new LinkedList<InterpreterContextRunner>(),
-        new InterpreterOutput(null));
   }
 
   @AfterClass

spark/interpreter/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java

@@ -17,13 +17,15 @@
 
 package org.apache.zeppelin.spark;
 
+
 import com.google.common.io.Files;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.python.PythonInterpreterTest;
 import org.junit.Test;
 
@@ -33,9 +35,10 @@ import java.util.Properties;
 
 import static org.mockito.Mockito.mock;
 
+
 public class PySparkInterpreterTest extends PythonInterpreterTest {
 
-  private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class);
+  private RemoteInterpreterEventClient mockRemoteEventClient = mock(RemoteInterpreterEventClient.class);
 
   @Override
   public void setUp() throws InterpreterException {
@@ -52,13 +55,18 @@ public class PySparkInterpreterTest extends PythonInterpreterTest {
     properties.setProperty("zeppelin.spark.test", "true");
     properties.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1");
 
-    InterpreterContext.set(getInterpreterContext(mockRemoteEventClient));
     // create interpreter group
     intpGroup = new InterpreterGroup();
     intpGroup.put("note", new LinkedList<Interpreter>());
 
+    InterpreterContext context = InterpreterContext.builder()
+        .setInterpreterOut(new InterpreterOutput(null))
+        .setIntpEventClient(mockRemoteEventClient)
+        .build();
+    InterpreterContext.set(context);
     LazyOpenInterpreter sparkInterpreter =
         new LazyOpenInterpreter(new SparkInterpreter(properties));
+
     intpGroup.get("note").add(sparkInterpreter);
     sparkInterpreter.setInterpreterGroup(intpGroup);
 
@@ -86,4 +94,10 @@ public class PySparkInterpreterTest extends PythonInterpreterTest {
     IPySparkInterpreterTest.testPySpark(interpreter, mockRemoteEventClient);
   }
 
+  @Override
+  protected InterpreterContext getInterpreterContext() {
+    InterpreterContext context = super.getInterpreterContext();
+    context.setIntpEventClient(mockRemoteEventClient);
+    return context;
+  }
 }

spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkRInterpreterTest.java

@@ -17,21 +17,16 @@
 
 package org.apache.zeppelin.spark;
 
-import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
-import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 
@@ -47,7 +42,7 @@ public class SparkRInterpreterTest {
 
   private SparkRInterpreter sparkRInterpreter;
   private SparkInterpreter sparkInterpreter;
-  private RemoteEventClient mockRemoteEventClient = mock(RemoteEventClient.class);
+  private RemoteInterpreterEventClient mockRemoteIntpEventClient = mock(RemoteInterpreterEventClient.class);
 
   @Before
   public void setUp() throws InterpreterException {
@@ -60,6 +55,8 @@ public class SparkRInterpreterTest {
     properties.setProperty("zeppelin.R.knitr", "true");
     properties.setProperty("spark.r.backendConnectionTimeout", "10");
 
+    InterpreterContext context = getInterpreterContext();
+    InterpreterContext.set(context);
     sparkRInterpreter = new SparkRInterpreter(properties);
     sparkInterpreter = new SparkInterpreter(properties);
 
@@ -70,7 +67,6 @@ public class SparkRInterpreterTest {
     sparkInterpreter.setInterpreterGroup(interpreterGroup);
 
     sparkRInterpreter.open();
-    sparkInterpreter.getZeppelinContext().setEventClient(mockRemoteEventClient);
   }
 
   @After
@@ -86,7 +82,7 @@ public class SparkRInterpreterTest {
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     assertTrue(result.message().get(0).getData().contains("2"));
     // spark web url is sent
-    verify(mockRemoteEventClient).onMetaInfosReceived(any(Map.class));
+    verify(mockRemoteIntpEventClient).onMetaInfosReceived(any(Map.class));
 
     result = sparkRInterpreter.interpret("sparkR.version()", getInterpreterContext());
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -96,7 +92,7 @@ public class SparkRInterpreterTest {
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
       // spark job url is sent
-      verify(mockRemoteEventClient, atLeastOnce()).onParaInfosReceived(any(String.class), any(String.class), any(Map.class));
+      verify(mockRemoteIntpEventClient, atLeastOnce()).onParaInfosReceived(any(Map.class));
 
       // cancel
       final InterpreterContext context = getInterpreterContext();
@@ -127,7 +123,7 @@ public class SparkRInterpreterTest {
       assertEquals(InterpreterResult.Code.SUCCESS, result.code());
       assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
       // spark job url is sent
-      verify(mockRemoteEventClient, atLeastOnce()).onParaInfosReceived(any(String.class), any(String.class), any(Map.class));
+      verify(mockRemoteIntpEventClient, atLeastOnce()).onParaInfosReceived(any(Map.class));
     }
 
     // plotting
@@ -155,8 +151,9 @@ public class SparkRInterpreterTest {
     InterpreterContext context = InterpreterContext.builder()
         .setNoteId("note_1")
         .setParagraphId("paragraph_1")
-        .setEventClient(mockRemoteEventClient)
+        .setIntpEventClient(mockRemoteIntpEventClient)
         .build();
     return context;
   }
 }
+

spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.spark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(Enclosed.class)
+public class SparkShimsTest {
+
+  @RunWith(Parameterized.class)
+  public static class ParamTests {
+    @Parameters(name = "Hadoop {0} supports jobUrl: {1}")
+    public static Collection<Object[]> data() {
+      return Arrays.asList(
+          new Object[][] {
+            {"2.6.0", false},
+            {"2.6.1", false},
+            {"2.6.2", false},
+            {"2.6.3", false},
+            {"2.6.4", false},
+            {"2.6.5", false},
+            {"2.6.6", true}, // The latest fixed version
+            {"2.6.7", true}, // Future version
+            {"2.7.0", false},
+            {"2.7.1", false},
+            {"2.7.2", false},
+            {"2.7.3", false},
+            {"2.7.4", true}, // The latest fixed version
+            {"2.7.5", true}, // Future versions
+            {"2.8.0", false},
+            {"2.8.1", false},
+            {"2.8.2", true}, // The latest fixed version
+            {"2.8.3", true}, // Future versions
+            {"2.9.0", true}, // The latest fixed version
+            {"2.9.1", true}, // Future versions
+            {"3.0.0", true}, // The latest fixed version
+            {"3.0.0-alpha4", true}, // The latest fixed version
+            {"3.0.1", true}, // Future versions
+          });
+    }
+
+    @Parameter public String version;
+
+    @Parameter(1)
+    public boolean expected;
+
+    @Test
+    public void checkYarnVersionTest() {
+      SparkShims sparkShims =
+          new SparkShims() {
+            @Override
+            public void setupSparkListener(String master,
+                                           String sparkWebUrl,
+                                           InterpreterContext context) {}
+          };
+      assertEquals(expected, sparkShims.supportYarn6615(version));
+    }
+  }
+
+  @RunWith(PowerMockRunner.class)
+  @PrepareForTest({BaseZeppelinContext.class, VersionInfo.class})
+  @PowerMockIgnore({"javax.net.*", "javax.security.*"})
+  public static class SingleTests {
+    @Mock Properties mockProperties;
+    @Captor ArgumentCaptor<Map<String, String>> argumentCaptor;
+
+    SparkShims sparkShims;
+    InterpreterContext mockContext;
+    RemoteInterpreterEventClient mockIntpEventClient;
+
+    @Before
+    public void setUp() {
+      mockContext = mock(InterpreterContext.class);
+      mockIntpEventClient = mock(RemoteInterpreterEventClient.class);
+      when(mockContext.getIntpEventClient()).thenReturn(mockIntpEventClient);
+      doNothing().when(mockIntpEventClient).onParaInfosReceived(argumentCaptor.capture());
+      try {
+        sparkShims = SparkShims.getInstance(SparkVersion.SPARK_2_0_0.toString());
+      } catch (Throwable ignore) {
+        sparkShims = SparkShims.getInstance(SparkVersion.SPARK_1_6_0.toString());
+      }
+    }
+
+    @Test
+    public void runUnderLocalTest() {
+      sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockProperties, mockContext);
+
+      Map<String, String> mapValue = argumentCaptor.getValue();
+      assertTrue(mapValue.keySet().contains("jobUrl"));
+      assertTrue(mapValue.get("jobUrl").contains("/jobs/job?id="));
+    }
+
+    @Test
+    public void runUnderYarnTest() {
+
+      sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockProperties, mockContext);
+
+      Map<String, String> mapValue = argumentCaptor.getValue();
+      assertTrue(mapValue.keySet().contains("jobUrl"));
+
+      if (sparkShims.supportYarn6615(VersionInfo.getVersion())) {
+        assertTrue(mapValue.get("jobUrl").contains("/jobs/job?id="));
+      } else {
+        assertFalse(mapValue.get("jobUrl").contains("/jobs/job?id="));
+      }
+    }
+  }
+}

spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala

@@ -35,8 +35,9 @@ import scala.tools.nsc.interpreter._
   * SparkInterpreter for scala-2.10
   */
 class SparkScala210Interpreter(override val conf: SparkConf,
-                               override val depFiles: java.util.List[String])
-  extends BaseSparkScalaInterpreter(conf, depFiles) {
+                               override val depFiles: java.util.List[String],
+                               override val printReplOutput: java.lang.Boolean)
+  extends BaseSparkScalaInterpreter(conf, depFiles, printReplOutput) {
 
   lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 
@@ -66,7 +67,9 @@ class SparkScala210Interpreter(override val conf: SparkConf,
     settings.embeddedDefaults(Thread.currentThread().getContextClassLoader())
     settings.usejavacp.value = true
     settings.classpath.value = getUserJars.mkString(File.pathSeparator)
-    Console.setOut(interpreterOutput)
+    if (printReplOutput) {
+      Console.setOut(interpreterOutput)
+    }
     sparkILoop = new SparkILoop()
 
     setDeclaredField(sparkILoop, "settings", settings)

spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala

@@ -35,8 +35,9 @@ import scala.tools.nsc.interpreter._
   * SparkInterpreter for scala-2.11
   */
 class SparkScala211Interpreter(override val conf: SparkConf,
-                               override val depFiles: java.util.List[String])
-  extends BaseSparkScalaInterpreter(conf, depFiles) {
+                               override val depFiles: java.util.List[String],
+                               override val printReplOutput: java.lang.Boolean)
+  extends BaseSparkScalaInterpreter(conf, depFiles, printReplOutput) {
 
   lazy override val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 
@@ -66,7 +67,11 @@ class SparkScala211Interpreter(override val conf: SparkConf,
     settings.usejavacp.value = true
     settings.classpath.value = getUserJars.mkString(File.pathSeparator)
 
-    val replOut = new JPrintWriter(interpreterOutput, true)
+    val replOut = if (printReplOutput) {
+      new JPrintWriter(interpreterOutput, true)
+    } else {
+      new JPrintWriter(Console.out, true)
+    }
     sparkILoop = new ILoop(None, replOut)
     sparkILoop.settings = settings
     sparkILoop.createInterpreter()

spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala

@@ -38,7 +38,8 @@ import scala.util.control.NonFatal
   * @param depFiles
   */
 abstract class BaseSparkScalaInterpreter(val conf: SparkConf,
-                                         val depFiles: java.util.List[String]) {
+                                         val depFiles: java.util.List[String],
+                                         val printReplOutput: java.lang.Boolean) {
 
   protected lazy val LOGGER: Logger = LoggerFactory.getLogger(getClass)
 

spark/spark-shims/pom.xml

@@ -41,6 +41,17 @@
       <version>${project.version}</version>
       <scope>provided</scope>
     </dependency>
+
+    <!--
+      This is for ZEPPELIN-2221 using VersionInfo for check the version of Yarn.
+      It's checked that VersionInfo is compatible at least 2.2.0 to the latest one.
+    -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>2.6.0</version>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
 
   <build>

spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java

@@ -15,12 +15,14 @@
  * limitations under the License.
  */
 
-
 package org.apache.zeppelin.spark;
 
-
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.VersionUtil;
 import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,11 +31,21 @@ import java.util.Map;
 import java.util.Properties;
 
 /**
- * This is abstract class for anything that is api incompatible between spark1 and spark2.
- * It will load the correct version of SparkShims based on the version of Spark.
+ * This is abstract class for anything that is api incompatible between spark1 and spark2. It will
+ * load the correct version of SparkShims based on the version of Spark.
  */
 public abstract class SparkShims {
 
+  // the following lines for checking specific versions
+  private static final String HADOOP_VERSION_2_6_6 = "2.6.6";
+  private static final String HADOOP_VERSION_2_7_0 = "2.7.0";
+  private static final String HADOOP_VERSION_2_7_4 = "2.7.4";
+  private static final String HADOOP_VERSION_2_8_0 = "2.8.0";
+  private static final String HADOOP_VERSION_2_8_2 = "2.8.2";
+  private static final String HADOOP_VERSION_2_9_0 = "2.9.0";
+  private static final String HADOOP_VERSION_3_0_0 = "3.0.0";
+  private static final String HADOOP_VERSION_3_0_0_ALPHA4 = "3.0.0-alpha4";
+
   private static final Logger LOGGER = LoggerFactory.getLogger(SparkShims.class);
 
   private static SparkShims sparkShims;
@@ -69,11 +81,12 @@ public abstract class SparkShims {
   }
 
   /**
-   * This is due to SparkListener api change between spark1 and spark2.
-   * SparkListener is trait in spark1 while it is abstract class in spark2.
+   * This is due to SparkListener api change between spark1 and spark2. SparkListener is trait in
+   * spark1 while it is abstract class in spark2.
    */
-  public abstract void setupSparkListener(String sparkWebUrl);
-
+  public abstract void setupSparkListener(String master,
+                                          String sparkWebUrl,
+                                          InterpreterContext context);
 
   protected String getNoteId(String jobgroupId) {
     int indexOf = jobgroupId.indexOf("-");
@@ -87,24 +100,44 @@ public abstract class SparkShims {
     return jobgroupId.substring(secondIndex + 1, jobgroupId.length());
   }
 
-  protected void buildSparkJobUrl(String sparkWebUrl, int jobId, Properties jobProperties) {
-    String jobGroupId = jobProperties.getProperty("spark.jobGroup.id");
+  protected void buildSparkJobUrl(String master,
+                                  String sparkWebUrl,
+                                  int jobId,
+                                  Properties jobProperties,
+                                  InterpreterContext context) {
     String uiEnabled = jobProperties.getProperty("spark.ui.enabled");
     String jobUrl = sparkWebUrl + "/jobs/job?id=" + jobId;
-    String noteId = getNoteId(jobGroupId);
-    String paragraphId = getParagraphId(jobGroupId);
     // Button visible if Spark UI property not set, set as invalid boolean or true
     boolean showSparkUI =
         uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false");
+    String version = VersionInfo.getVersion();
+    if (master.toLowerCase().contains("yarn") && !supportYarn6615(version)) {
+      jobUrl = sparkWebUrl + "/jobs";
+    }
     if (showSparkUI && jobUrl != null) {
-      RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient();
       Map<String, String> infos = new java.util.HashMap<String, String>();
       infos.put("jobUrl", jobUrl);
       infos.put("label", "SPARK JOB");
       infos.put("tooltip", "View in Spark web UI");
-      if (eventClient != null) {
-        eventClient.onParaInfosReceived(noteId, paragraphId, infos);
-      }
+      context.getIntpEventClient().onParaInfosReceived(infos);
     }
   }
+
+  /**
+   * This is temporal patch for support old versions of Yarn which is not adopted YARN-6615
+   *
+   * @return true if YARN-6615 is patched, false otherwise
+   */
+  protected boolean supportYarn6615(String version) {
+    return (VersionUtil.compareVersions(HADOOP_VERSION_2_6_6, version) <= 0
+            && VersionUtil.compareVersions(HADOOP_VERSION_2_7_0, version) > 0)
+        || (VersionUtil.compareVersions(HADOOP_VERSION_2_7_4, version) <= 0
+            && VersionUtil.compareVersions(HADOOP_VERSION_2_8_0, version) > 0)
+        || (VersionUtil.compareVersions(HADOOP_VERSION_2_8_2, version) <= 0
+            && VersionUtil.compareVersions(HADOOP_VERSION_2_9_0, version) > 0)
+        || (VersionUtil.compareVersions(HADOOP_VERSION_2_9_0, version) <= 0
+            && VersionUtil.compareVersions(HADOOP_VERSION_3_0_0, version) > 0)
+        || (VersionUtil.compareVersions(HADOOP_VERSION_3_0_0_ALPHA4, version) <= 0)
+        || (VersionUtil.compareVersions(HADOOP_VERSION_3_0_0, version) <= 0);
+  }
 }

spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java

@@ -19,38 +19,20 @@
 package org.apache.zeppelin.spark;
 
 import org.apache.spark.SparkContext;
-import org.apache.spark.scheduler.SparkListener;
-import org.apache.spark.scheduler.SparkListenerApplicationEnd;
-import org.apache.spark.scheduler.SparkListenerApplicationStart;
-import org.apache.spark.scheduler.SparkListenerBlockManagerAdded;
-import org.apache.spark.scheduler.SparkListenerBlockManagerRemoved;
-import org.apache.spark.scheduler.SparkListenerBlockUpdated;
-import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate;
-import org.apache.spark.scheduler.SparkListenerExecutorAdded;
-import org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate;
-import org.apache.spark.scheduler.SparkListenerExecutorRemoved;
-import org.apache.spark.scheduler.SparkListenerJobEnd;
 import org.apache.spark.scheduler.SparkListenerJobStart;
-import org.apache.spark.scheduler.SparkListenerStageCompleted;
-import org.apache.spark.scheduler.SparkListenerStageSubmitted;
-import org.apache.spark.scheduler.SparkListenerTaskEnd;
-import org.apache.spark.scheduler.SparkListenerTaskGettingResult;
-import org.apache.spark.scheduler.SparkListenerTaskStart;
-import org.apache.spark.scheduler.SparkListenerUnpersistRDD;
 import org.apache.spark.ui.jobs.JobProgressListener;
-import org.apache.zeppelin.interpreter.BaseZeppelinContext;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
-
-import java.util.Map;
+import org.apache.zeppelin.interpreter.InterpreterContext;
 
 public class Spark1Shims extends SparkShims {
 
-  public void setupSparkListener(final String sparkWebUrl) {
+  public void setupSparkListener(final String master,
+                                 final String sparkWebUrl,
+                                 final InterpreterContext context) {
     SparkContext sc = SparkContext.getOrCreate();
     sc.addSparkListener(new JobProgressListener(sc.getConf()) {
       @Override
       public void onJobStart(SparkListenerJobStart jobStart) {
-        buildSparkJobUrl(sparkWebUrl, jobStart.jobId(), jobStart.properties());
+        buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
       }
     });
   }

spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java

@@ -21,15 +21,18 @@ package org.apache.zeppelin.spark;
 import org.apache.spark.SparkContext;
 import org.apache.spark.scheduler.SparkListener;
 import org.apache.spark.scheduler.SparkListenerJobStart;
+import org.apache.zeppelin.interpreter.InterpreterContext;
 
 public class Spark2Shims extends SparkShims {
 
-  public void setupSparkListener(final String sparkWebUrl) {
+  public void setupSparkListener(final String master,
+                                 final String sparkWebUrl,
+                                 final InterpreterContext context) {
     SparkContext sc = SparkContext.getOrCreate();
     sc.addSparkListener(new SparkListener() {
       @Override
       public void onJobStart(SparkListenerJobStart jobStart) {
-        buildSparkJobUrl(sparkWebUrl, jobStart.jobId(), jobStart.properties());
+        buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context);
       }
     });
   }

testing/install_external_dependencies.sh

@@ -45,6 +45,6 @@ if [[ -n "$PYTHON" ]] ; then
   conda update -q conda
   conda info -a
   conda config --add channels conda-forge
-  conda install -q matplotlib=2.1.2 pandasql ipython=5.4.1 jupyter_client ipykernel matplotlib bokeh=0.12.10
-  pip install -q grpcio ggplot bkzep==0.4.0 statsmodels==0.8.0
+  conda install -q pandas=0.21.1 matplotlib=2.1.1 pandasql=0.7.3 ipython=5.4.1 jupyter_client=5.1.0 ipykernel=4.7.0 bokeh=0.12.10
+  pip install -q ggplot==0.11.5 grpcio==1.8.2 bkzep==0.4.0
 fi

zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularElemTest.scala

@@ -34,12 +34,11 @@ trait AbstractAngularElemTest
 
   override def beforeEach() {
     val intpGroup = new InterpreterGroup()
-    val context = new InterpreterContext("note", "paragraph", null, "title", "text",
-      new AuthenticationInfo(), new util.HashMap[String, Object](), new GUI(), new GUI(),
-      new AngularObjectRegistry(intpGroup.getId(), null),
-      null,
-      new util.LinkedList[InterpreterContextRunner](),
-      new InterpreterOutput(null));
+    val context = InterpreterContext.builder
+      .setNoteId("noteId")
+      .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+      .setInterpreterOut(new InterpreterOutput(null))
+      .build()
 
     InterpreterContext.set(context)
     super.beforeEach() // To be stackable, must call super.beforeEach

zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularModelTest.scala

@@ -29,12 +29,11 @@ trait AbstractAngularModelTest extends FlatSpec
 with BeforeAndAfter with BeforeAndAfterEach with Eventually with Matchers {
   override def beforeEach() {
     val intpGroup = new InterpreterGroup()
-    val context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
-      new java.util.HashMap[String, Object](), new GUI(), new GUI(), new AngularObjectRegistry(
-        intpGroup.getId(), null),
-      null,
-      new java.util.LinkedList[InterpreterContextRunner](),
-      new InterpreterOutput(null));
+    val context = InterpreterContext.builder
+      .setNoteId("noteId")
+      .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null))
+      .setInterpreterOut(new InterpreterOutput(null))
+      .build()
 
     InterpreterContext.set(context)
     super.beforeEach() // To be stackable, must call super.beforeEach

zeppelin-distribution/src/bin_license/LICENSE

@@ -38,7 +38,7 @@ The following components are provided under Apache License.
     (Apache 2.0) Apache Lens (http://lens.apache.org/)
     (Apache 2.0) Apache Flink (http://flink.apache.org/)
     (Apache 2.0) Apache Beam (http://beam.apache.org/)
-    (Apache 2.0) Apache Thrift 0.9.2 (org.apache.thrift:libthrift:0.9.2 - http://thrift.apache.org/)
+    (Apache 2.0) Apache Thrift 0.9.3 (org.apache.thrift:libthrift:0.9.3 - http://thrift.apache.org/)
     (Apache 2.0) Apache Lucene (https://lucene.apache.org/)
     (Apache 2.0) Apache Zookeeper (org.apache.zookeeper:zookeeper:jar:3.4.5 - http://zookeeper.apache.org/)
     (Apache 2.0) Chill (com.twitter:chill:0.8.0 - https://github.com/twitter/chill/)

zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java

@@ -63,14 +63,14 @@ public class AuthenticationIT extends AbstractZeppelinIT {
       "securityManager.sessionManager = $sessionManager\n" +
       "securityManager.sessionManager.globalSessionTimeout = 86400000\n" +
       "shiro.loginUrl = /api/login\n" +
-      "anyofroles = org.apache.zeppelin.utils.AnyOfRolesAuthorizationFilter\n" +
+      "anyofrolesuser = org.apache.zeppelin.utils.AnyOfRolesUserAuthorizationFilter\n" +
       "[roles]\n" +
       "admin = *\n" +
       "hr = *\n" +
       "finance = *\n" +
       "[urls]\n" +
       "/api/version = anon\n" +
-      "/api/interpreter/** = authc, anyofroles[admin, finance]\n" +
+      "/api/interpreter/** = authc, anyofrolesuser[admin, finance]\n" +
       "/** = authc";
 
   static String originalShiro = "";
@@ -172,7 +172,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
   }
 
   @Test
-  public void testAnyOfRoles() throws Exception {
+  public void testAnyOfRolesUser() throws Exception {
     try {
       AuthenticationIT authenticationIT = new AuthenticationIT();
       authenticationIT.authenticationUser("admin", "password1");
@@ -220,7 +220,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
       authenticationIT.logoutUser("hr1");
 
     } catch (Exception e) {
-      handleException("Exception in AuthenticationIT while testAnyOfRoles ", e);
+      handleException("Exception in AuthenticationIT while testAnyOfRolesUser ", e);
     }
   }
 

zeppelin-integration/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java

@@ -236,7 +236,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
     }
   }
 
-  @Test
+//  @Test
   public void testRunOnSelectionChange() throws Exception {
     try {
       String xpathToRunOnSelectionChangeCheckbox = getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]";
@@ -480,7 +480,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
     }
   }
 
-  @Test
+//  @Test
   public void testEditOnDoubleClick() throws Exception {
     try {
       createNewNote();
@@ -712,7 +712,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
     }
   }
 
-  @Test
+  // @Test
   public void testNoteDynamicFormSelect() throws Exception {
     try {
       createNewNote();

zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -501,12 +501,12 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     }
   }
 
-  public String getCallbackPortRange() {
-    return getString(ConfVars.ZEPPELIN_INTERPRETER_CALLBACK_PORTRANGE);
+  public String getZeppelinServerRPCPortRange() {
+    return getString(ConfVars.ZEPPELIN_SERVER_RPC_PORTRANGE);
   }
 
   public String getInterpreterPortRange() {
-    return getString(ConfVars.ZEPPELIN_INTERPRETER_PORTRANGE);
+    return getString(ConfVars.ZEPPELIN_INTERPRETER_RPC_PORTRANGE);
   }
 
   public boolean isWindowsPath(String path){
@@ -607,6 +607,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_CRON_FOLDERS);
   }
 
+  public Boolean isZeppelinNotebookCollaborativeModeEnable() {
+    return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_COLLABORATIVE_MODE_ENABLE);
+  }
+
   public String getZeppelinProxyUrl() {
     return getString(ConfVars.ZEPPELIN_PROXY_URL);
   }
@@ -724,7 +728,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
     ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
         "http://repo1.maven.org/maven2/"),
-    ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
+    ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 60000),
     ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
     ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
         + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
@@ -797,8 +801,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_SERVER_KERBEROS_KEYTAB("zeppelin.server.kerberos.keytab", ""),
     ZEPPELIN_SERVER_KERBEROS_PRINCIPAL("zeppelin.server.kerberos.principal", ""),
 
-    ZEPPELIN_INTERPRETER_CALLBACK_PORTRANGE("zeppelin.interpreter.callback.portRange", ":"),
-    ZEPPELIN_INTERPRETER_PORTRANGE("zeppelin.interpreter.portRange", ":"),
+    ZEPPELIN_SERVER_RPC_PORTRANGE("zeppelin.server.rpc.portRange", ":"),
+    ZEPPELIN_INTERPRETER_RPC_PORTRANGE("zeppelin.interpreter.rpc.portRange", ":"),
 
     ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_CLASS("zeppelin.interpreter.lifecyclemanager.class",
         "org.apache.zeppelin.interpreter.lifecycle.NullLifecycleManager"),
@@ -813,6 +817,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME("zeppelin.notebook.git.remote.username", "token"),
     ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN("zeppelin.notebook.git.remote.access-token", ""),
     ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN("zeppelin.notebook.git.remote.origin", "origin"),
+    ZEPPELIN_NOTEBOOK_COLLABORATIVE_MODE_ENABLE("zeppelin.notebook.collaborative.mode.enable",
+            true),
     ZEPPELIN_NOTEBOOK_CRON_ENABLE("zeppelin.notebook.cron.enable", false),
     ZEPPELIN_NOTEBOOK_CRON_FOLDERS("zeppelin.notebook.cron.folders", null),
     ZEPPELIN_PROXY_URL("zeppelin.proxy.url", null),

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java

@@ -17,17 +17,17 @@
 
 package org.apache.zeppelin.display;
 
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ExecutorService;
-
 import com.google.gson.Gson;
 import org.apache.zeppelin.common.JsonSerializable;
 import org.apache.zeppelin.scheduler.ExecutorFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
 /**
  * AngularObject provides binding between back-end (interpreter) and front-end
  * User provided object will automatically synchronized with front-end side.
@@ -150,7 +150,7 @@ public class AngularObject<T> implements JsonSerializable {
    * fire updated() event for listener
    * Note that it does not invoke watcher.watch()
    */
-  public void emit(){
+  public void emit() {
     if (listener != null) {
       listener.updated(this);
     }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java

@@ -17,6 +17,7 @@
 
 package org.apache.zeppelin.interpreter;
 
+import org.apache.thrift.TException;
 import org.apache.zeppelin.annotation.Experimental;
 import org.apache.zeppelin.annotation.ZeppelinApi;
 import org.apache.zeppelin.display.AngularObject;
@@ -24,7 +25,6 @@ import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.AngularObjectWatcher;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
 import org.apache.zeppelin.resource.Resource;
 import org.apache.zeppelin.resource.ResourcePool;
 import org.apache.zeppelin.resource.ResourceSet;
@@ -32,6 +32,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -50,8 +51,6 @@ public abstract class BaseZeppelinContext {
   protected GUI gui;
   protected GUI noteGui;
 
-  private static RemoteEventClientWrapper eventClient;
-
   public BaseZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
     this.hooks = hooks;
     this.maxResult = maxResult;
@@ -69,6 +68,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * subclasses should implement this method to display specific data type
+   *
    * @param obj
    * @return
    */
@@ -76,7 +76,6 @@ public abstract class BaseZeppelinContext {
 
   /**
    * @deprecated use z.textbox instead
-   *
    */
   @Deprecated
   @ZeppelinApi
@@ -147,7 +146,7 @@ public abstract class BaseZeppelinContext {
 
 
   private Object select(String name, Object defaultValue, ParamOption[] paramOptions,
-                       boolean noteForm) {
+                        boolean noteForm) {
     if (noteForm) {
       return noteGui.select(name, defaultValue, paramOptions);
     } else {
@@ -164,7 +163,7 @@ public abstract class BaseZeppelinContext {
   }
 
   private Collection<Object> checkbox(String name, ParamOption[] options,
-                                     boolean noteForm) {
+                                      boolean noteForm) {
     List<Object> defaultValues = new LinkedList<>();
     for (ParamOption option : options) {
       defaultValues.add(option.getValue());
@@ -217,6 +216,7 @@ public abstract class BaseZeppelinContext {
   /**
    * display special types of objects for interpreter.
    * Each interpreter can has its own supported classes.
+   *
    * @param o object
    */
   @ZeppelinApi
@@ -227,7 +227,8 @@ public abstract class BaseZeppelinContext {
   /**
    * display special types of objects for interpreter.
    * Each interpreter can has its own supported classes.
-   * @param o object
+   *
+   * @param o         object
    * @param maxResult maximum number of rows to display
    */
 
@@ -257,208 +258,127 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Run paragraph by id
-   * @param noteId
+   *
    * @param paragraphId
    */
   @ZeppelinApi
-  public void run(String noteId, String paragraphId) {
-    run(noteId, paragraphId, interpreterContext, true);
-  }
-
-  /**
-   * Run paragraph by id
-   * @param paragraphId
-   */
-  @ZeppelinApi
-  public void run(String paragraphId) {
+  public void run(String paragraphId) throws IOException {
     run(paragraphId, true);
   }
 
   /**
    * Run paragraph by id
+   *
    * @param paragraphId
    * @param checkCurrentParagraph
    */
   @ZeppelinApi
-  public void run(String paragraphId, boolean checkCurrentParagraph) {
+  public void run(String paragraphId, boolean checkCurrentParagraph) throws IOException {
     String noteId = interpreterContext.getNoteId();
     run(noteId, paragraphId, interpreterContext, checkCurrentParagraph);
   }
 
+  @ZeppelinApi
+  public void run(String noteId, String paragraphId)
+      throws IOException {
+    run(noteId, paragraphId, InterpreterContext.get(), true);
+  }
+
   /**
    * Run paragraph by id
+   *
    * @param noteId
    */
   @ZeppelinApi
-  public void run(String noteId, String paragraphId, InterpreterContext context) {
+  public void run(String noteId, String paragraphId, InterpreterContext context)
+      throws IOException {
     run(noteId, paragraphId, context, true);
   }
 
   /**
    * Run paragraph by id
+   *
    * @param noteId
    * @param context
    */
   @ZeppelinApi
   public void run(String noteId, String paragraphId, InterpreterContext context,
-                  boolean checkCurrentParagraph) {
+                  boolean checkCurrentParagraph) throws IOException {
+
     if (paragraphId.equals(context.getParagraphId()) && checkCurrentParagraph) {
       throw new RuntimeException("Can not run current Paragraph");
     }
-
-    List<InterpreterContextRunner> runners =
-        getInterpreterContextRunner(noteId, paragraphId, context);
-
-    if (runners.size() <= 0) {
-      throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size());
-    }
-
-    for (InterpreterContextRunner r : runners) {
-      r.run();
-    }
-
+    List<String> paragraphIds = new ArrayList<>();
+    paragraphIds.add(paragraphId);
+    List<Integer> paragraphIndices = new ArrayList<>();
+    context.getIntpEventClient()
+        .runParagraphs(noteId, paragraphIds, paragraphIndices, context.getParagraphId());
   }
 
-  public void runNote(String noteId) {
+  public void runNote(String noteId) throws IOException {
     runNote(noteId, interpreterContext);
   }
 
-  public void runNote(String noteId, InterpreterContext context) {
-    String runningNoteId = context.getNoteId();
-    String runningParagraphId = context.getParagraphId();
-    List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
-
-    if (runners.size() <= 0) {
-      throw new RuntimeException("Note " + noteId + " not found " + runners.size());
-    }
-
-    for (InterpreterContextRunner r : runners) {
-      if (r.getNoteId().equals(runningNoteId) && r.getParagraphId().equals(runningParagraphId)) {
-        continue;
-      }
-      LOGGER.debug("Run Paragraph: " + r.getParagraphId() + " of Note: " + r.getNoteId());
-      r.run();
-    }
-  }
-
-
-  /**
-   * get Zeppelin Paragraph Runner from zeppelin server
-   * @param noteId
-   */
-  @ZeppelinApi
-  public List<InterpreterContextRunner> getInterpreterContextRunner(
-      String noteId, InterpreterContext interpreterContext) {
-    List<InterpreterContextRunner> runners = new LinkedList<>();
-    RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
-
-    if (remoteWorksController != null) {
-      runners = remoteWorksController.getRemoteContextRunner(noteId);
-    }
-
-    return runners;
-  }
-
-  /**
-   * get Zeppelin Paragraph Runner from zeppelin server
-   * @param noteId
-   * @param paragraphId
-   */
-  @ZeppelinApi
-  public List<InterpreterContextRunner> getInterpreterContextRunner(
-      String noteId, String paragraphId, InterpreterContext interpreterContext) {
-    List<InterpreterContextRunner> runners = new LinkedList<>();
-    RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
-
-    if (remoteWorksController != null) {
-      runners = remoteWorksController.getRemoteContextRunner(noteId, paragraphId);
-    }
-
-    return runners;
+  public void runNote(String noteId, InterpreterContext context) throws IOException {
+    List<String> paragraphIds = new ArrayList<>();
+    List<Integer> paragraphIndices = new ArrayList<>();
+    context.getIntpEventClient()
+        .runParagraphs(noteId, paragraphIds, paragraphIndices, context.getParagraphId());
   }
 
   /**
    * Run paragraph at idx
+   *
    * @param idx
    */
   @ZeppelinApi
-  public void run(int idx) {
+  public void run(int idx) throws IOException {
     run(idx, true);
   }
 
   /**
-   *
-   * @param idx  paragraph index
-   * @param checkCurrentParagraph  check whether you call this run method in the current paragraph.
-   * Set it to false only when you are sure you are not invoking this method to run current
-   * paragraph. Otherwise you would run current paragraph in infinite loop.
+   * @param idx                   paragraph index
+   * @param checkCurrentParagraph check whether you call this run method in the current paragraph.
+   *          Set it to false only when you are sure you are not invoking this method to run current
+   *          paragraph. Otherwise you would run current paragraph in infinite loop.
    */
-  public void run(int idx, boolean checkCurrentParagraph) {
+  public void run(int idx, boolean checkCurrentParagraph) throws IOException {
     String noteId = interpreterContext.getNoteId();
     run(noteId, idx, interpreterContext, checkCurrentParagraph);
   }
 
   /**
    * Run paragraph at index
+   *
    * @param noteId
-   * @param idx index starting from 0
+   * @param idx     index starting from 0
    * @param context interpreter context
    */
-  public void run(String noteId, int idx, InterpreterContext context) {
+  public void run(String noteId, int idx, InterpreterContext context) throws IOException {
     run(noteId, idx, context, true);
   }
 
   /**
-   *
    * @param noteId
-   * @param idx  paragraph index
-   * @param context interpreter context
-   * @param checkCurrentParagraph check whether you call this run method in the current paragraph.
+   * @param idx                   paragraph index
+   * @param context               interpreter context
+   * @param checkCurrentParagraph
+   * check whether you call this run method in the current paragraph.
    * Set it to false only when you are sure you are not invoking this method to run current
    * paragraph. Otherwise you would run current paragraph in infinite loop.
    */
   public void run(String noteId, int idx, InterpreterContext context,
-                  boolean checkCurrentParagraph) {
-    List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
-    if (idx >= runners.size()) {
-      throw new RuntimeException("Index out of bound");
-    }
+                  boolean checkCurrentParagraph) throws IOException {
 
-    InterpreterContextRunner runner = runners.get(idx);
-    if (runner.getParagraphId().equals(context.getParagraphId()) && checkCurrentParagraph) {
-      throw new RuntimeException("Can not run current Paragraph: " + runner.getParagraphId());
-    }
-
-    runner.run();
+    List<String> paragraphIds = new ArrayList<>();
+    List<Integer> paragraphIndices = new ArrayList<>();
+    paragraphIndices.add(idx);
+    context.getIntpEventClient()
+        .runParagraphs(noteId, paragraphIds, paragraphIndices, context.getParagraphId());
   }
 
   @ZeppelinApi
-  public void run(List<Object> paragraphIdOrIdx) {
-    run(paragraphIdOrIdx, interpreterContext);
-  }
-
-  /**
-   * Run paragraphs
-   * @param paragraphIdOrIdx list of paragraph id or idx
-   */
-  @ZeppelinApi
-  public void run(List<Object> paragraphIdOrIdx, InterpreterContext context) {
-    String noteId = context.getNoteId();
-    for (Object idOrIdx : paragraphIdOrIdx) {
-      if (idOrIdx instanceof String) {
-        String paragraphId = (String) idOrIdx;
-        run(noteId, paragraphId, context);
-      } else if (idOrIdx instanceof Integer) {
-        Integer idx = (Integer) idOrIdx;
-        run(noteId, idx, context);
-      } else {
-        throw new RuntimeException("Paragraph " + idOrIdx + " not found");
-      }
-    }
-  }
-
-  @ZeppelinApi
-  public void runAll() {
+  public void runAll() throws IOException {
     runAll(interpreterContext);
   }
 
@@ -466,22 +386,10 @@ public abstract class BaseZeppelinContext {
    * Run all paragraphs. except this.
    */
   @ZeppelinApi
-  public void runAll(InterpreterContext context) {
+  public void runAll(InterpreterContext context) throws IOException {
     runNote(context.getNoteId());
   }
 
-  @ZeppelinApi
-  public List<String> listParagraphs() {
-    List<String> paragraphs = new LinkedList<>();
-
-    for (InterpreterContextRunner r : interpreterContext.getRunners()) {
-      paragraphs.add(r.getParagraphId());
-    }
-
-    return paragraphs;
-  }
-
-
   private AngularObject getAngularObject(String name, InterpreterContext interpreterContext) {
     AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
     String noteId = interpreterContext.getNoteId();
@@ -501,6 +409,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Get angular object. Look up notebook scope first and then global scope
+   *
    * @param name variable name
    * @return value
    */
@@ -516,6 +425,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Get angular object. Look up global scope
+   *
    * @param name variable name
    * @return value
    */
@@ -533,52 +443,58 @@ public abstract class BaseZeppelinContext {
   /**
    * Create angular variable in notebook scope and bind with front end Angular display system.
    * If variable exists, it'll be overwritten.
+   *
    * @param name name of the variable
-   * @param o value
+   * @param o    value
    */
   @ZeppelinApi
-  public void angularBind(String name, Object o) {
+  public void angularBind(String name, Object o) throws TException {
     angularBind(name, o, interpreterContext.getNoteId());
   }
 
   /**
    * Create angular variable in global scope and bind with front end Angular display system.
    * If variable exists, it'll be overwritten.
+   *
    * @param name name of the variable
-   * @param o value
+   * @param o    value
    */
   @Deprecated
-  public void angularBindGlobal(String name, Object o) {
+  public void angularBindGlobal(String name, Object o) throws TException {
     angularBind(name, o, (String) null);
   }
 
   /**
    * Create angular variable in local scope and bind with front end Angular display system.
    * If variable exists, value will be overwritten and watcher will be added.
-   * @param name name of variable
-   * @param o value
+   *
+   * @param name    name of variable
+   * @param o       value
    * @param watcher watcher of the variable
    */
   @ZeppelinApi
-  public void angularBind(String name, Object o, AngularObjectWatcher watcher) {
+  public void angularBind(String name, Object o, AngularObjectWatcher watcher) throws TException {
     angularBind(name, o, interpreterContext.getNoteId(), watcher);
   }
 
   /**
    * Create angular variable in global scope and bind with front end Angular display system.
    * If variable exists, value will be overwritten and watcher will be added.
-   * @param name name of variable
-   * @param o value
+   *
+   * @param name    name of variable
+   * @param o       value
    * @param watcher watcher of the variable
    */
   @Deprecated
-  public void angularBindGlobal(String name, Object o, AngularObjectWatcher watcher) {
+  public void angularBindGlobal(String name, Object o, AngularObjectWatcher watcher)
+      throws TException {
     angularBind(name, o, null, watcher);
   }
 
   /**
    * Add watcher into angular variable (local scope)
-   * @param name name of the variable
+   *
+   * @param name    name of the variable
    * @param watcher watcher
    */
   @ZeppelinApi
@@ -588,7 +504,8 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Add watcher into angular variable (global scope)
-   * @param name name of the variable
+   *
+   * @param name    name of the variable
    * @param watcher watcher
    */
   @Deprecated
@@ -597,9 +514,9 @@ public abstract class BaseZeppelinContext {
   }
 
 
-
   /**
    * Remove watcher from angular variable (local)
+   *
    * @param name
    * @param watcher
    */
@@ -610,6 +527,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove watcher from angular variable (global)
+   *
    * @param name
    * @param watcher
    */
@@ -621,6 +539,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove all watchers for the angular variable (local)
+   *
    * @param name
    */
   @ZeppelinApi
@@ -630,6 +549,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove all watchers for the angular variable (global)
+   *
    * @param name
    */
   @Deprecated
@@ -639,30 +559,33 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove angular variable and all the watchers.
+   *
    * @param name
    */
   @ZeppelinApi
-  public void angularUnbind(String name) {
+  public void angularUnbind(String name) throws TException {
     String noteId = interpreterContext.getNoteId();
     angularUnbind(name, noteId);
   }
 
   /**
    * Remove angular variable and all the watchers.
+   *
    * @param name
    */
   @Deprecated
-  public void angularUnbindGlobal(String name) {
+  public void angularUnbindGlobal(String name) throws TException {
     angularUnbind(name, null);
   }
 
   /**
    * Create angular variable in notebook scope and bind with front end Angular display system.
    * If variable exists, it'll be overwritten.
+   *
    * @param name name of the variable
-   * @param o value
+   * @param o    value
    */
-  private void angularBind(String name, Object o, String noteId) {
+  public void angularBind(String name, Object o, String noteId) throws TException {
     AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
 
     if (registry.get(name, noteId, null) == null) {
@@ -676,11 +599,13 @@ public abstract class BaseZeppelinContext {
    * Create angular variable in notebook scope and bind with front end Angular display
    * system.
    * If variable exists, value will be overwritten and watcher will be added.
-   * @param name name of variable
-   * @param o value
+   *
+   * @param name    name of variable
+   * @param o       value
    * @param watcher watcher of the variable
    */
-  private void angularBind(String name, Object o, String noteId, AngularObjectWatcher watcher) {
+  private void angularBind(String name, Object o, String noteId, AngularObjectWatcher watcher)
+      throws TException {
     AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
 
     if (registry.get(name, noteId, null) == null) {
@@ -693,7 +618,8 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Add watcher into angular binding variable
-   * @param name name of the variable
+   *
+   * @param name    name of the variable
    * @param watcher watcher
    */
   public void angularWatch(String name, String noteId, AngularObjectWatcher watcher) {
@@ -706,6 +632,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove watcher
+   *
    * @param name
    * @param watcher
    */
@@ -718,6 +645,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove all watchers for the angular variable
+   *
    * @param name
    */
   private void angularUnwatch(String name, String noteId) {
@@ -729,15 +657,17 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove angular variable and all the watchers.
+   *
    * @param name
    */
-  private void angularUnbind(String name, String noteId) {
+  private void angularUnbind(String name, String noteId) throws TException {
     AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
     registry.remove(name, noteId, null);
   }
 
   /**
    * Get the interpreter class name from name entered in paragraph
+   *
    * @param replName if replName is a valid className, return that instead.
    */
   private String getClassNameFromReplName(String replName) {
@@ -750,8 +680,9 @@ public abstract class BaseZeppelinContext {
 
   /**
    * General function to register hook event
-   * @param event The type of event to hook to (pre_exec, post_exec)
-   * @param cmd The code to be executed by the interpreter on given event
+   *
+   * @param event    The type of event to hook to (pre_exec, post_exec)
+   * @param cmd      The code to be executed by the interpreter on given event
    * @param replName Name of the interpreter
    */
   @Experimental
@@ -762,8 +693,9 @@ public abstract class BaseZeppelinContext {
 
   /**
    * registerHook() wrapper for current repl
+   *
    * @param event The type of event to hook to (pre_exec, post_exec)
-   * @param cmd The code to be executed by the interpreter on given event
+   * @param cmd   The code to be executed by the interpreter on given event
    */
   @Experimental
   public void registerHook(String event, String cmd) throws InvalidHookException {
@@ -772,7 +704,6 @@ public abstract class BaseZeppelinContext {
   }
 
   /**
-   *
    * @param event
    * @param cmd
    * @param noteId
@@ -795,7 +726,7 @@ public abstract class BaseZeppelinContext {
   /**
    * Unbind code from given hook event and given repl
    *
-   * @param event The type of event to hook to (pre_exec, post_exec)
+   * @param event    The type of event to hook to (pre_exec, post_exec)
    * @param replName Name of the interpreter
    */
   @Experimental
@@ -806,6 +737,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * unregisterHook() wrapper for current repl
+   *
    * @param event The type of event to hook to (pre_exec, post_exec)
    */
   @Experimental
@@ -816,8 +748,8 @@ public abstract class BaseZeppelinContext {
   /**
    * Unbind code from given hook event and given note
    *
-   * @param noteId  The id of note
-   * @param event The type of event to hook to (pre_exec, post_exec)
+   * @param noteId The id of note
+   * @param event  The type of event to hook to (pre_exec, post_exec)
    */
   @Experimental
   public void unregisterNoteHook(String noteId, String event) {
@@ -828,8 +760,9 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Unbind code from given hook event, given note and given repl
-   * @param noteId  The id of note
-   * @param event The type of event to hook to (pre_exec, post_exec)
+   *
+   * @param noteId   The id of note
+   * @param event    The type of event to hook to (pre_exec, post_exec)
    * @param replName Name of the interpreter
    */
   @Experimental
@@ -841,6 +774,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Add object into resource pool
+   *
    * @param name
    * @param value
    */
@@ -853,6 +787,7 @@ public abstract class BaseZeppelinContext {
   /**
    * Get object from resource pool
    * Search local process first and then the other processes
+   *
    * @param name
    * @return null if resource not found
    */
@@ -869,6 +804,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Remove object from resourcePool
+   *
    * @param name
    */
   @ZeppelinApi
@@ -879,6 +815,7 @@ public abstract class BaseZeppelinContext {
 
   /**
    * Check if resource pool has the object
+   *
    * @param name
    * @return
    */
@@ -897,22 +834,4 @@ public abstract class BaseZeppelinContext {
     ResourcePool resourcePool = interpreterContext.getResourcePool();
     return resourcePool.getAll();
   }
-
-  /**
-   * Get the event client
-   */
-  @ZeppelinApi
-  public static RemoteEventClientWrapper getEventClient() {
-    return eventClient;
-  }
-
-  /**
-   * Set event client
-   */
-  @ZeppelinApi
-  public void setEventClient(RemoteEventClientWrapper eventClient) {
-    if (BaseZeppelinContext.eventClient == null) {
-      BaseZeppelinContext.eventClient = eventClient;
-    }
-  }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java

@@ -17,18 +17,14 @@
 
 package org.apache.zeppelin.interpreter;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.user.AuthenticationInfo;
 import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
-import org.apache.zeppelin.interpreter.remote.RemoteEventClient;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient;
 import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * Interpreter context
@@ -61,17 +57,19 @@ public class InterpreterContext {
   private GUI noteGui = new GUI();
   private AngularObjectRegistry angularObjectRegistry;
   private ResourcePool resourcePool;
-  private List<InterpreterContextRunner> runners = new ArrayList<>();
   private String interpreterClassName;
-  private RemoteEventClientWrapper client;
-  private RemoteWorksController remoteWorksController;
   private Map<String, Integer> progressMap;
+  private RemoteInterpreterEventClient intpEventClient;
 
   /**
    * Builder class for InterpreterContext
    */
   public static class Builder {
-    private InterpreterContext context = new InterpreterContext();
+    private InterpreterContext context;
+
+    public Builder() {
+      context = new InterpreterContext();
+    }
 
     public Builder setNoteId(String noteId) {
       context.noteId = noteId;
@@ -83,13 +81,18 @@ public class InterpreterContext {
       return this;
     }
 
-    public Builder setEventClient(RemoteEventClientWrapper client) {
-      context.client = client;
+    public Builder setInterpreterClassName(String intpClassName) {
+      context.interpreterClassName = intpClassName;
       return this;
     }
 
-    public Builder setInterpreterClassName(String intpClassName) {
-      context.interpreterClassName = intpClassName;
+    public Builder setAngularObjectRegistry(AngularObjectRegistry angularObjectRegistry) {
+      context.angularObjectRegistry = angularObjectRegistry;
+      return this;
+    }
+
+    public Builder setResourcePool(ResourcePool resourcePool) {
+      context.resourcePool = resourcePool;
       return this;
     }
 
@@ -98,7 +101,54 @@ public class InterpreterContext {
       return this;
     }
 
+    public Builder setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+      context.authenticationInfo = authenticationInfo;
+      return this;
+    }
+
+    public Builder setConfig(Map<String, Object> config) {
+      context.config = config;
+      return this;
+    }
+
+    public Builder setGUI(GUI gui) {
+      context.gui = gui;
+      return this;
+    }
+
+    public Builder setNoteGUI(GUI noteGUI) {
+      context.noteGui = noteGUI;
+      return this;
+    }
+
+    public Builder setInterpreterOut(InterpreterOutput out) {
+      context.out = out;
+      return this;
+    }
+
+    public Builder setIntpEventClient(RemoteInterpreterEventClient intpEventClient) {
+      context.intpEventClient = intpEventClient;
+      return this;
+    }
+
+    public Builder setProgressMap(Map<String, Integer> progressMap) {
+      context.progressMap = progressMap;
+      return this;
+    }
+
+    public Builder setParagraphText(String paragraphText) {
+      context.paragraphText = paragraphText;
+      return this;
+    }
+
+    public Builder setParagraphTitle(String paragraphTitle) {
+      context.paragraphTitle = paragraphTitle;
+      return this;
+    }
+
+
     public InterpreterContext build() {
+      InterpreterContext.set(context);
       return context;
     }
   }
@@ -111,79 +161,6 @@ public class InterpreterContext {
 
   }
 
-  // visible for testing
-  public InterpreterContext(String noteId,
-                            String paragraphId,
-                            String replName,
-                            String paragraphTitle,
-                            String paragraphText,
-                            AuthenticationInfo authenticationInfo,
-                            Map<String, Object> config,
-                            GUI gui,
-                            GUI noteGui,
-                            AngularObjectRegistry angularObjectRegistry,
-                            ResourcePool resourcePool,
-                            List<InterpreterContextRunner> runners,
-                            InterpreterOutput out
-                            ) {
-    this(noteId, paragraphId, replName, paragraphTitle, paragraphText, authenticationInfo,
-        config, gui, noteGui, angularObjectRegistry, resourcePool, runners, out, null, null);
-  }
-
-  public InterpreterContext(String noteId,
-                            String paragraphId,
-                            String replName,
-                            String paragraphTitle,
-                            String paragraphText,
-                            AuthenticationInfo authenticationInfo,
-                            Map<String, Object> config,
-                            GUI gui,
-                            GUI noteGui,
-                            AngularObjectRegistry angularObjectRegistry,
-                            ResourcePool resourcePool,
-                            List<InterpreterContextRunner> runners,
-                            InterpreterOutput out,
-                            RemoteWorksController remoteWorksController,
-                            Map<String, Integer> progressMap
-                            ) {
-    this.noteId = noteId;
-    this.paragraphId = paragraphId;
-    this.replName = replName;
-    this.paragraphTitle = paragraphTitle;
-    this.paragraphText = paragraphText;
-    this.authenticationInfo = authenticationInfo;
-    this.config = config;
-    this.gui = gui;
-    this.noteGui = noteGui;
-    this.angularObjectRegistry = angularObjectRegistry;
-    this.resourcePool = resourcePool;
-    this.runners = runners;
-    this.out = out;
-    this.remoteWorksController = remoteWorksController;
-    this.progressMap = progressMap;
-  }
-
-  public InterpreterContext(String noteId,
-                            String paragraphId,
-                            String replName,
-                            String paragraphTitle,
-                            String paragraphText,
-                            AuthenticationInfo authenticationInfo,
-                            Map<String, Object> config,
-                            GUI gui,
-                            GUI noteGui,
-                            AngularObjectRegistry angularObjectRegistry,
-                            ResourcePool resourcePool,
-                            List<InterpreterContextRunner> contextRunners,
-                            InterpreterOutput output,
-                            RemoteWorksController remoteWorksController,
-                            RemoteInterpreterEventClient eventClient,
-                            Map<String, Integer> progressMap) {
-    this(noteId, paragraphId, replName, paragraphTitle, paragraphText, authenticationInfo,
-        config, gui, noteGui, angularObjectRegistry, resourcePool, contextRunners, output,
-        remoteWorksController, progressMap);
-    this.client = new RemoteEventClient(eventClient);
-  }
 
   public String getNoteId() {
     return noteId;
@@ -229,10 +206,6 @@ public class InterpreterContext {
     return resourcePool;
   }
 
-  public List<InterpreterContextRunner> getRunners() {
-    return runners;
-  }
-
   public String getInterpreterClassName() {
     return interpreterClassName;
   }
@@ -241,20 +214,12 @@ public class InterpreterContext {
     this.interpreterClassName = className;
   }
 
-  public RemoteEventClientWrapper getClient() {
-    return client;
+  public RemoteInterpreterEventClient getIntpEventClient() {
+    return intpEventClient;
   }
 
-  public void setClient(RemoteEventClientWrapper client) {
-    this.client = client;
-  }
-
-  public RemoteWorksController getRemoteWorksController() {
-    return remoteWorksController;
-  }
-
-  public void setRemoteWorksController(RemoteWorksController remoteWorksController) {
-    this.remoteWorksController = remoteWorksController;
+  public void setIntpEventClient(RemoteInterpreterEventClient intpEventClient) {
+    this.intpEventClient = intpEventClient;
   }
 
   public InterpreterOutput out() {

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java

@@ -17,6 +17,8 @@
 
 package org.apache.zeppelin.interpreter.launcher;
 
+import java.io.IOException;
+
 /**
  * Interface to InterpreterClient which is created by InterpreterLauncher. This is the component
  * that is used to for the communication from zeppelin-server process to zeppelin interpreter
@@ -26,7 +28,7 @@ public interface InterpreterClient {
 
   String getInterpreterSettingName();
 
-  void start(String userName);
+  void start(String userName) throws IOException;
 
   void stop();
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java

@@ -35,6 +35,8 @@ public class InterpreterLaunchContext {
   private String interpreterSettingId;
   private String interpreterSettingGroup;
   private String interpreterSettingName;
+  private int zeppelinServerRPCPort;
+  private String zeppelinServerHost;
 
   public InterpreterLaunchContext(Properties properties,
                                   InterpreterOption option,
@@ -43,7 +45,9 @@ public class InterpreterLaunchContext {
                                   String interpreterGroupId,
                                   String interpreterSettingId,
                                   String interpreterSettingGroup,
-                                  String interpreterSettingName) {
+                                  String interpreterSettingName,
+                                  int zeppelinServerRPCPort,
+                                  String zeppelinServerHost) {
     this.properties = properties;
     this.option = option;
     this.runner = runner;
@@ -52,6 +56,8 @@ public class InterpreterLaunchContext {
     this.interpreterSettingId = interpreterSettingId;
     this.interpreterSettingGroup = interpreterSettingGroup;
     this.interpreterSettingName = interpreterSettingName;
+    this.zeppelinServerRPCPort = zeppelinServerRPCPort;
+    this.zeppelinServerHost = zeppelinServerHost;
   }
 
   public Properties getProperties() {
@@ -85,4 +91,12 @@ public class InterpreterLaunchContext {
   public String getUserName() {
     return userName;
   }
+
+  public int getZeppelinServerRPCPort() {
+    return zeppelinServerRPCPort;
+  }
+
+  public String getZeppelinServerHost() {
+    return zeppelinServerHost;
+  }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java

@@ -37,5 +37,16 @@ public abstract class InterpreterLauncher {
     this.recoveryStorage = recoveryStorage;
   }
 
-  public abstract  InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
+  protected int getConnectTimeout() {
+    int connectTimeout =
+        zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+    if (properties.containsKey(
+        ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName())) {
+      connectTimeout = Integer.parseInt(properties.getProperty(
+          ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName()));
+    }
+    return connectTimeout;
+  }
+
+  public abstract InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteEventClient.java

@@ -1,34 +0,0 @@
-package org.apache.zeppelin.interpreter.remote;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * 
- * Wrapper arnd RemoteInterpreterEventClient
- * to expose methods in the client
- *
- */
-public class RemoteEventClient implements RemoteEventClientWrapper {
-
-  private RemoteInterpreterEventClient client;
-
-  public RemoteEventClient(RemoteInterpreterEventClient client) {
-    this.client = client;
-  }
-
-  @Override
-  public void onMetaInfosReceived(Map<String, String> infos) {
-    client.onMetaInfosReceived(infos);
-  }
-
-  @Override
-  public void onParaInfosReceived(String noteId, String paragraphId, Map<String, String> infos) {
-    Map<String, String> paraInfos =  new HashMap<String, String>(infos);
-    paraInfos.put("noteId", noteId);
-    paraInfos.put("paraId", paragraphId);
-    client.onParaInfosReceived(paraInfos);
-  }
-
-
-}

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java

@@ -17,14 +17,19 @@
 package org.apache.zeppelin.interpreter.remote;
 
 import com.google.gson.Gson;
+import org.apache.thrift.TException;
 import org.apache.zeppelin.display.AngularObject;
-import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
-import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
-import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
+import org.apache.zeppelin.interpreter.thrift.AppOutputAppendEvent;
+import org.apache.zeppelin.interpreter.thrift.AppOutputUpdateEvent;
+import org.apache.zeppelin.interpreter.thrift.AppStatusUpdateEvent;
+import org.apache.zeppelin.interpreter.thrift.OutputAppendEvent;
+import org.apache.zeppelin.interpreter.thrift.OutputUpdateAllEvent;
+import org.apache.zeppelin.interpreter.thrift.OutputUpdateEvent;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventService;
+import org.apache.zeppelin.interpreter.thrift.RunParagraphsEvent;
 import org.apache.zeppelin.resource.RemoteResource;
 import org.apache.zeppelin.resource.Resource;
 import org.apache.zeppelin.resource.ResourceId;
@@ -35,141 +40,67 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.LinkedList;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 /**
- * Thread connection ZeppelinServer -> RemoteInterpreterServer does not provide
- * remote method invocation from RemoteInterpreterServer -> ZeppelinServer
- *
- * This class provides event send and get response from RemoteInterpreterServer to
- * ZeppelinServer.
- *
- * RemoteInterpreterEventPoller is counter part in ZeppelinServer
+ * This class is used to communicate with ZeppelinServer via thrift.
+ * All the methods are synchronized because thrift client is not thread safe.
  */
-public class RemoteInterpreterEventClient implements ResourcePoolConnector {
-  private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventClient.class);
-  private final List<RemoteInterpreterEvent> eventQueue = new LinkedList<>();
-  private final List<ResourceSet> getAllResourceResponse = new LinkedList<>();
-  private final Map<ResourceId, Object> getResourceResponse = new HashMap<>();
-  private final Map<InvokeResourceMethodEventMessage, Object> getInvokeResponse = new HashMap<>();
+public class RemoteInterpreterEventClient implements ResourcePoolConnector,
+    AngularObjectRegistryListener {
+  private final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreterEventClient.class);
   private final Gson gson = new Gson();
 
-  /**
-   * Run paragraph
-   * @param runner
-   */
-  public void getZeppelinServerNoteRunner(
-      String eventOwnerKey, ZeppelinServerResourceParagraphRunner runner) {
-    RemoteZeppelinServerResource eventBody = new RemoteZeppelinServerResource();
-    eventBody.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
-    eventBody.setOwnerKey(eventOwnerKey);
-    eventBody.setData(runner);
+  private RemoteInterpreterEventService.Client intpEventServiceClient;
+  private String intpGroupId;
 
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE,
-        gson.toJson(eventBody)));
+  public RemoteInterpreterEventClient(RemoteInterpreterEventService.Client intpEventServiceClient) {
+    this.intpEventServiceClient = intpEventServiceClient;
   }
 
-  /**
-   * Run paragraph
-   * @param runner
-   */
-  public void run(InterpreterContextRunner runner) {
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER,
-        gson.toJson(runner)));
+  public void setIntpGroupId(String intpGroupId) {
+    this.intpGroupId = intpGroupId;
   }
 
-  /**
-   * notify new angularObject creation
-   * @param object
-   */
-  public void angularObjectAdd(AngularObject object) {
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.ANGULAR_OBJECT_ADD, object.toJson()));
-  }
-
-  /**
-   * notify angularObject update
-   */
-  public void angularObjectUpdate(AngularObject object) {
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE, object.toJson()));
-  }
-
-  /**
-   * notify angularObject removal
-   */
-  public void angularObjectRemove(String name, String noteId, String paragraphId) {
-    Map<String, String> removeObject = new HashMap<>();
-    removeObject.put("name", name);
-    removeObject.put("noteId", noteId);
-    removeObject.put("paragraphId", paragraphId);
-
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE, gson.toJson(removeObject)));
-  }
-
-
   /**
    * Get all resources except for specific resourcePool
+   *
    * @return
    */
   @Override
-  public ResourceSet getAllResources() {
-    // request
-    sendEvent(new RemoteInterpreterEvent(RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL, null));
-
-    synchronized (getAllResourceResponse) {
-      while (getAllResourceResponse.isEmpty()) {
-        try {
-          getAllResourceResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
+  public synchronized ResourceSet getAllResources() {
+    try {
+      List<String> resources = intpEventServiceClient.getAllResources(intpGroupId);
+      ResourceSet resourceSet = new ResourceSet();
+      for (String res : resources) {
+        RemoteResource resource = RemoteResource.fromJson(res);
+        resource.setResourcePoolConnector(this);
+        resourceSet.add(resource);
       }
-      ResourceSet resourceSet = getAllResourceResponse.remove(0);
       return resourceSet;
+    } catch (TException e) {
+      LOGGER.warn("Fail to getAllResources", e);
+      return null;
     }
   }
 
   @Override
-  public Object readResource(ResourceId resourceId) {
-    logger.debug("Request Read Resource {} from ZeppelinServer", resourceId.getName());
-    synchronized (getResourceResponse) {
-      // wait for previous response consumed
-      while (getResourceResponse.containsKey(resourceId)) {
-        try {
-          getResourceResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-
-      // send request
-      sendEvent(new RemoteInterpreterEvent(
-          RemoteInterpreterEventType.RESOURCE_GET,
-          resourceId.toJson()));
-
-      // wait for response
-      while (!getResourceResponse.containsKey(resourceId)) {
-        try {
-          getResourceResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-      Object o = getResourceResponse.remove(resourceId);
-      getResourceResponse.notifyAll();
+  public synchronized Object readResource(ResourceId resourceId) {
+    try {
+      ByteBuffer buffer = intpEventServiceClient.getResource(resourceId.toJson());
+      Object o = Resource.deserializeObject(buffer);
       return o;
+    } catch (TException | IOException | ClassNotFoundException e) {
+      LOGGER.warn("Failt to readResource: " + resourceId, e);
+      return null;
     }
   }
 
   /**
    * Invoke method and save result in resourcePool as another resource
+   *
    * @param resourceId
    * @param methodName
    * @param paramTypes
@@ -177,49 +108,51 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
    * @return
    */
   @Override
-  public Object invokeMethod(
+  public synchronized Object invokeMethod(
       ResourceId resourceId,
       String methodName,
       Class[] paramTypes,
       Object[] params) {
-    logger.debug("Request Invoke method {} of Resource {}", methodName, resourceId.getName());
+    LOGGER.debug("Request Invoke method {} of Resource {}", methodName, resourceId.getName());
 
-    InvokeResourceMethodEventMessage invokeMethod = new InvokeResourceMethodEventMessage(
-        resourceId,
-        methodName,
-        paramTypes,
-        params,
-        null);
-
-    synchronized (getInvokeResponse) {
-      // wait for previous response consumed
-      while (getInvokeResponse.containsKey(invokeMethod)) {
-        try {
-          getInvokeResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-      // send request
-      sendEvent(new RemoteInterpreterEvent(
-          RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD,
-          invokeMethod.toJson()));
-      // wait for response
-      while (!getInvokeResponse.containsKey(invokeMethod)) {
-        try {
-          getInvokeResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-      Object o = getInvokeResponse.remove(invokeMethod);
-      getInvokeResponse.notifyAll();
-      return o;
-    }
+    return null;
+    //    InvokeResourceMethodEventMessage invokeMethod = new InvokeResourceMethodEventMessage(
+    //        resourceId,
+    //        methodName,
+    //        paramTypes,
+    //        params,
+    //        null);
+    //
+    //    synchronized (getInvokeResponse) {
+    //      // wait for previous response consumed
+    //      while (getInvokeResponse.containsKey(invokeMethod)) {
+    //        try {
+    //          getInvokeResponse.wait();
+    //        } catch (InterruptedException e) {
+    //          LOGGER.warn(e.getMessage(), e);
+    //        }
+    //      }
+    //      // send request
+    //      sendEvent(new RemoteInterpreterEvent(
+    //          RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD,
+    //          invokeMethod.toJson()));
+    //      // wait for response
+    //      while (!getInvokeResponse.containsKey(invokeMethod)) {
+    //        try {
+    //          getInvokeResponse.wait();
+    //        } catch (InterruptedException e) {
+    //          LOGGER.warn(e.getMessage(), e);
+    //        }
+    //      }
+    //      Object o = getInvokeResponse.remove(invokeMethod);
+    //      getInvokeResponse.notifyAll();
+    //      return o;
+    //    }
   }
 
   /**
    * Invoke method and save result in resourcePool as another resource
+   *
    * @param resourceId
    * @param methodName
    * @param paramTypes
@@ -228,267 +161,180 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
    * @return
    */
   @Override
-  public Resource invokeMethod(
+  public synchronized Resource invokeMethod(
       ResourceId resourceId,
       String methodName,
       Class[] paramTypes,
       Object[] params,
       String returnResourceName) {
-    logger.debug("Request Invoke method {} of Resource {}", methodName, resourceId.getName());
+    LOGGER.debug("Request Invoke method {} of Resource {}", methodName, resourceId.getName());
 
-    InvokeResourceMethodEventMessage invokeMethod = new InvokeResourceMethodEventMessage(
-        resourceId,
-        methodName,
-        paramTypes,
-        params,
-        returnResourceName);
-
-    synchronized (getInvokeResponse) {
-      // wait for previous response consumed
-      while (getInvokeResponse.containsKey(invokeMethod)) {
-        try {
-          getInvokeResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-      // send request
-      sendEvent(new RemoteInterpreterEvent(
-          RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD,
-          invokeMethod.toJson()));
-      // wait for response
-      while (!getInvokeResponse.containsKey(invokeMethod)) {
-        try {
-          getInvokeResponse.wait();
-        } catch (InterruptedException e) {
-          logger.warn(e.getMessage(), e);
-        }
-      }
-      Resource o = (Resource) getInvokeResponse.remove(invokeMethod);
-      getInvokeResponse.notifyAll();
-      return o;
-    }
+    return null;
+    //    InvokeResourceMethodEventMessage invokeMethod = new InvokeResourceMethodEventMessage(
+    //        resourceId,
+    //        methodName,
+    //        paramTypes,
+    //        params,
+    //        returnResourceName);
+    //
+    //    synchronized (getInvokeResponse) {
+    //      // wait for previous response consumed
+    //      while (getInvokeResponse.containsKey(invokeMethod)) {
+    //        try {
+    //          getInvokeResponse.wait();
+    //        } catch (InterruptedException e) {
+    //          LOGGER.warn(e.getMessage(), e);
+    //        }
+    //      }
+    //      // send request
+    //      sendEvent(new RemoteInterpreterEvent(
+    //          RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD,
+    //          invokeMethod.toJson()));
+    //      // wait for response
+    //      while (!getInvokeResponse.containsKey(invokeMethod)) {
+    //        try {
+    //          getInvokeResponse.wait();
+    //        } catch (InterruptedException e) {
+    //          LOGGER.warn(e.getMessage(), e);
+    //        }
+    //      }
+    //      Resource o = (Resource) getInvokeResponse.remove(invokeMethod);
+    //      getInvokeResponse.notifyAll();
+    //      return o;
+    //    }
   }
 
-  /**
-   * Supposed to call from RemoteInterpreterEventPoller
-   */
-  public void putResponseGetAllResources(List<String> resources) {
-    logger.debug("ResourceSet from ZeppelinServer");
-    ResourceSet resourceSet = new ResourceSet();
-
-    for (String res : resources) {
-      RemoteResource resource = RemoteResource.fromJson(res);
-      resource.setResourcePoolConnector(this);
-      resourceSet.add(resource);
-    }
-
-    synchronized (getAllResourceResponse) {
-      getAllResourceResponse.add(resourceSet);
-      getAllResourceResponse.notify();
-    }
-  }
-
-  /**
-   * Supposed to call from RemoteInterpreterEventPoller
-   * @param resourceId json serialized ResourceId
-   * @param object java serialized of the object
-   */
-  public void putResponseGetResource(String resourceId, ByteBuffer object) {
-    ResourceId rid = ResourceId.fromJson(resourceId);
-
-    logger.debug("Response resource {} from RemoteInterpreter", rid.getName());
-
-    Object o = null;
-    try {
-      o = Resource.deserializeObject(object);
-    } catch (IOException e) {
-      logger.error(e.getMessage(), e);
-    } catch (ClassNotFoundException e) {
-      logger.error(e.getMessage(), e);
-    }
-
-    synchronized (getResourceResponse) {
-      getResourceResponse.put(rid, o);
-      getResourceResponse.notifyAll();
-    }
-  }
-
-
-  /**
-   * Supposed to call from RemoteInterpreterEventPoller
-   * @param invokeMessage json serialized InvokeMessage
-   * @param object java serialized of the object
-   */
-  public void putResponseInvokeMethod(
-      InvokeResourceMethodEventMessage invokeMessage, ByteBuffer object) {
-    Object o = null;
-    try {
-      o = Resource.deserializeObject(object);
-    } catch (IOException e) {
-      logger.error(e.getMessage(), e);
-    } catch (ClassNotFoundException e) {
-      logger.error(e.getMessage(), e);
-    }
-
-    synchronized (getInvokeResponse) {
-      getInvokeResponse.put(invokeMessage, o);
-      getInvokeResponse.notifyAll();
-    }
-  }
-
-  /**
-   * Supposed to call from RemoteInterpreterEventPoller
-   * @param invokeMessage invoke message
-   * @param resource remote resource
-   */
-  public void putResponseInvokeMethod(
-      InvokeResourceMethodEventMessage invokeMessage, Resource resource) {
-    synchronized (getInvokeResponse) {
-      getInvokeResponse.put(invokeMessage, resource);
-      getInvokeResponse.notifyAll();
-    }
-  }
-
-  /**
-   * Supposed to call from RemoteInterpreterEventPoller
-   * @return next available event
-   */
-  public RemoteInterpreterEvent pollEvent() {
-    synchronized (eventQueue) {
-      if (eventQueue.isEmpty()) {
-        try {
-          eventQueue.wait(1000);
-        } catch (InterruptedException e) {
-          // ignore exception
-        }
-      }
-
-      if (eventQueue.isEmpty()) {
-        return new RemoteInterpreterEvent(RemoteInterpreterEventType.NO_OP, "");
-      } else {
-        RemoteInterpreterEvent event = eventQueue.remove(0);
-        logger.debug("Send event {}", event.getType());
-        return event;
-      }
-    }
-  }
-
-  public void onInterpreterOutputAppend(
+  public synchronized void onInterpreterOutputAppend(
       String noteId, String paragraphId, int outputIndex, String output) {
-    Map<String, String> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("index", Integer.toString(outputIndex));
-    appendOutput.put("data", output);
-
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.OUTPUT_APPEND,
-        gson.toJson(appendOutput)));
+    try {
+      intpEventServiceClient.appendOutput(
+          new OutputAppendEvent(noteId, paragraphId, outputIndex, output, null));
+    } catch (TException e) {
+      LOGGER.warn("Fail to appendOutput", e);
+    }
   }
 
-  public void onInterpreterOutputUpdate(
+  public synchronized void onInterpreterOutputUpdate(
       String noteId, String paragraphId, int outputIndex,
       InterpreterResult.Type type, String output) {
-    Map<String, String> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("index", Integer.toString(outputIndex));
-    appendOutput.put("type", type.name());
-    appendOutput.put("data", output);
-
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.OUTPUT_UPDATE,
-        gson.toJson(appendOutput)));
-  }
-
-  public void onInterpreterOutputUpdateAll(
-      String noteId, String paragraphId, List<InterpreterResultMessage> messages) {
-    Map<String, Object> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("messages", messages);
-
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.OUTPUT_UPDATE_ALL,
-        gson.toJson(appendOutput)));
-  }
-
-  private void sendEvent(RemoteInterpreterEvent event) {
-    logger.debug("Send Event: " + event);
-    synchronized (eventQueue) {
-      eventQueue.add(event);
-      eventQueue.notifyAll();
+    try {
+      intpEventServiceClient.updateOutput(
+          new OutputUpdateEvent(noteId, paragraphId, outputIndex, type.name(), output, null));
+    } catch (TException e) {
+      LOGGER.warn("Fail to updateOutput", e);
     }
   }
 
-  public void onAppOutputAppend(
-      String noteId, String paragraphId, int index, String appId, String output) {
-    Map<String, Object> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("index", Integer.toString(index));
-    appendOutput.put("appId", appId);
-    appendOutput.put("data", output);
+  public synchronized void onInterpreterOutputUpdateAll(
+      String noteId, String paragraphId, List<InterpreterResultMessage> messages) {
+    try {
+      intpEventServiceClient.updateAllOutput(
+          new OutputUpdateAllEvent(noteId, paragraphId, convertToThrift(messages)));
+    } catch (TException e) {
+      LOGGER.warn("Fail to updateAllOutput", e);
+    }
+  }
 
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.OUTPUT_APPEND,
-        gson.toJson(appendOutput)));
+  private List<org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage>
+        convertToThrift(List<InterpreterResultMessage> messages) {
+    List<org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage> thriftMessages =
+        new ArrayList<>();
+    for (InterpreterResultMessage message : messages) {
+      thriftMessages.add(
+          new org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage(
+              message.getType().name(), message.getData()));
+    }
+    return thriftMessages;
+  }
+
+  public synchronized void runParagraphs(String noteId,
+                                         List<String> paragraphIds,
+                                         List<Integer> paragraphIndices,
+                                         String curParagraphId) {
+    RunParagraphsEvent event =
+        new RunParagraphsEvent(noteId, paragraphIds, paragraphIndices, curParagraphId);
+    try {
+      intpEventServiceClient.runParagraphs(event);
+    } catch (TException e) {
+      LOGGER.warn("Fail to runParagraphs: " + event, e);
+    }
+  }
+
+  public synchronized void onAppOutputAppend(
+      String noteId, String paragraphId, int index, String appId, String output) {
+    AppOutputAppendEvent event =
+        new AppOutputAppendEvent(noteId, paragraphId, appId, index, output);
+    try {
+      intpEventServiceClient.appendAppOutput(event);
+    } catch (TException e) {
+      LOGGER.warn("Fail to appendAppOutput: " + event, e);
+    }
   }
 
 
-  public void onAppOutputUpdate(
+  public synchronized void onAppOutputUpdate(
       String noteId, String paragraphId, int index, String appId,
       InterpreterResult.Type type, String output) {
-    Map<String, Object> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("index", Integer.toString(index));
-    appendOutput.put("appId", appId);
-    appendOutput.put("type", type);
-    appendOutput.put("data", output);
-    logger.debug("onAppoutputUpdate = {}", output);
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.OUTPUT_UPDATE,
-        gson.toJson(appendOutput)));
+    AppOutputUpdateEvent event =
+        new AppOutputUpdateEvent(noteId, paragraphId, appId, index, type.name(), output);
+    try {
+      intpEventServiceClient.updateAppOutput(event);
+    } catch (TException e) {
+      LOGGER.warn("Fail to updateAppOutput: " + event, e);
+    }
   }
 
-  public void onAppStatusUpdate(String noteId, String paragraphId, String appId, String status) {
-    Map<String, String> appendOutput = new HashMap<>();
-    appendOutput.put("noteId", noteId);
-    appendOutput.put("paragraphId", paragraphId);
-    appendOutput.put("appId", appId);
-    appendOutput.put("status", status);
-
-    sendEvent(new RemoteInterpreterEvent(
-        RemoteInterpreterEventType.APP_STATUS_UPDATE,
-        gson.toJson(appendOutput)));
+  public synchronized void onAppStatusUpdate(String noteId, String paragraphId, String appId,
+                                             String status) {
+    AppStatusUpdateEvent event = new AppStatusUpdateEvent(noteId, paragraphId, appId, status);
+    try {
+      intpEventServiceClient.updateAppStatus(event);
+    } catch (TException e) {
+      LOGGER.warn("Fail to updateAppStatus: " + event, e);
+    }
   }
 
-  public void onMetaInfosReceived(Map<String, String> infos) {
-    sendEvent(new RemoteInterpreterEvent(RemoteInterpreterEventType.META_INFOS,
-        gson.toJson(infos)));
+  public synchronized void onMetaInfosReceived(Map<String, String> infos) {
+    try {
+      intpEventServiceClient.sendMetaInfo(intpGroupId, gson.toJson(infos));
+    } catch (TException e) {
+      LOGGER.warn("Fail to sendMetaInfo: " + infos, e);
+    }
   }
 
-  public void onParaInfosReceived(Map<String, String> infos) {
-    sendEvent(new RemoteInterpreterEvent(RemoteInterpreterEventType.PARA_INFOS,
-        gson.toJson(infos)));
+  public synchronized void onParaInfosReceived(Map<String, String> infos) {
+    try {
+      intpEventServiceClient.sendParagraphInfo(intpGroupId, gson.toJson(infos));
+    } catch (TException e) {
+      LOGGER.warn("Fail to onParaInfosReceived: " + infos, e);
+    }
   }
-  /**
-   * Wait for eventQueue becomes empty
-   */
-  public void waitForEventQueueBecomesEmpty(long atMost) {
-    long startTime = System.currentTimeMillis();
-    synchronized (eventQueue) {
-      while (!eventQueue.isEmpty() && (System.currentTimeMillis() - startTime) < atMost) {
-        try {
-          eventQueue.wait(100);
-        } catch (InterruptedException e) {
-          // ignore exception
-        }
-      }
-      if (!eventQueue.isEmpty())
-        eventQueue.clear();
+
+  @Override
+  public synchronized void onAdd(String interpreterGroupId, AngularObject object) {
+    try {
+      intpEventServiceClient.addAngularObject(intpGroupId, object.toJson());
+    } catch (TException e) {
+      LOGGER.warn("Fail to add AngularObject: " + object, e);
+    }
+  }
+
+  @Override
+  public synchronized void onUpdate(String interpreterGroupId, AngularObject object) {
+    try {
+      intpEventServiceClient.updateAngularObject(intpGroupId, object.toJson());
+    } catch (TException e) {
+      LOGGER.warn("Fail to update AngularObject: " + object, e);
+    }
+  }
+
+  @Override
+  public synchronized void onRemove(String interpreterGroupId, String name, String noteId,
+                                    String paragraphId) {
+    try {
+      intpEventServiceClient.removeAngularObject(intpGroupId, noteId, paragraphId, name);
+    } catch (TException e) {
+      LOGGER.warn("Fail to remove AngularObject", e);
     }
   }
 }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -20,13 +20,16 @@ package org.apache.zeppelin.interpreter.remote;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
 import org.apache.thrift.server.TThreadPoolServer;
 import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
 import org.apache.thrift.transport.TTransportException;
 import org.apache.zeppelin.dep.DependencyResolver;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
-import org.apache.zeppelin.display.AngularObjectRegistryListener;
 import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.helium.Application;
 import org.apache.zeppelin.helium.ApplicationContext;
@@ -50,17 +53,15 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
 import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
-import org.apache.zeppelin.interpreter.RemoteWorksController;
 import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
-import org.apache.zeppelin.interpreter.thrift.CallbackInfo;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.interpreter.thrift.RegisterInfo;
 import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
+import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventService;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
-import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
 import org.apache.zeppelin.resource.DistributedResourcePool;
 import org.apache.zeppelin.resource.Resource;
 import org.apache.zeppelin.resource.ResourcePool;
@@ -97,33 +98,31 @@ import java.util.concurrent.ConcurrentMap;
  * Accepting thrift connections from ZeppelinServer.
  */
 public class RemoteInterpreterServer extends Thread
-    implements RemoteInterpreterService.Iface, AngularObjectRegistryListener {
+    implements RemoteInterpreterService.Iface {
 
   private static Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class);
 
-  InterpreterGroup interpreterGroup;
-  AngularObjectRegistry angularObjectRegistry;
-  InterpreterHookRegistry hookRegistry;
-  DistributedResourcePool resourcePool;
+  private String interpreterGroupId;
+  private InterpreterGroup interpreterGroup;
+  private AngularObjectRegistry angularObjectRegistry;
+  private InterpreterHookRegistry hookRegistry;
+  private DistributedResourcePool resourcePool;
   private ApplicationLoader appLoader;
+  private Gson gson = new Gson();
 
-  Gson gson = new Gson();
-
-  RemoteInterpreterService.Processor<RemoteInterpreterServer> processor;
-  private String callbackHost;
-  private int callbackPort;
+  private String intpEventServerHost;
   private String host;
   private int port;
   private TThreadPoolServer server;
+  RemoteInterpreterEventService.Client intpEventServiceClient;
 
-  RemoteInterpreterEventClient eventClient = new RemoteInterpreterEventClient();
+  RemoteInterpreterEventClient intpEventClient;
   private DependencyResolver depLoader;
 
   private final Map<String, RunningApplication> runningApplications =
       Collections.synchronizedMap(new HashMap<String, RunningApplication>());
 
   private Map<String, Object> remoteWorksResponsePool;
-  private ZeppelinRemoteWorksController remoteWorksController;
 
   private final long DEFAULT_SHUTDOWN_TIMEOUT = 2000;
 
@@ -132,27 +131,41 @@ public class RemoteInterpreterServer extends Thread
 
   private boolean isTest;
 
-  public RemoteInterpreterServer(String callbackHost, int callbackPort, String portRange)
+  public RemoteInterpreterServer(String intpEventServerHost,
+                                 int intpEventServerPort,
+                                 String interpreterGroupId,
+                                 String portRange)
       throws IOException, TTransportException {
-    this(callbackHost, callbackPort, portRange, false);
+    this(intpEventServerHost, intpEventServerPort, portRange, interpreterGroupId, false);
   }
 
-  public RemoteInterpreterServer(String callbackHost, int callbackPort, String portRange,
-                                 boolean isTest) throws TTransportException, IOException {
-    if (null != callbackHost) {
-      this.callbackHost = callbackHost;
-      this.callbackPort = callbackPort;
+  public RemoteInterpreterServer(String intpEventServerHost,
+                                 int intpEventServerPort,
+                                 String portRange,
+                                 String interpreterGroupId,
+                                 boolean isTest)
+      throws TTransportException, IOException {
+    if (null != intpEventServerHost) {
+      this.intpEventServerHost = intpEventServerHost;
+      if (!isTest) {
+        TTransport transport = new TSocket(intpEventServerHost, intpEventServerPort);
+        transport.open();
+        TProtocol protocol = new TBinaryProtocol(transport);
+        intpEventServiceClient = new RemoteInterpreterEventService.Client(protocol);
+        intpEventClient = new RemoteInterpreterEventClient(intpEventServiceClient);
+      }
     } else {
       // DevInterpreter
-      this.port = callbackPort;
+      this.port = intpEventServerPort;
     }
     this.isTest = isTest;
-
-    processor = new RemoteInterpreterService.Processor<>(this);
+    this.interpreterGroupId = interpreterGroupId;
+    RemoteInterpreterService.Processor<RemoteInterpreterServer> processor =
+        new RemoteInterpreterService.Processor<>(this);
     TServerSocket serverTransport;
-    if (null == callbackHost) {
+    if (null == intpEventServerHost) {
       // Dev Interpreter
-      serverTransport = new TServerSocket(callbackPort);
+      serverTransport = new TServerSocket(intpEventServerPort);
     } else {
       serverTransport = RemoteInterpreterUtils.createTServerSocket(portRange);
       this.port = serverTransport.getServerSocket().getLocalPort();
@@ -163,12 +176,11 @@ public class RemoteInterpreterServer extends Thread
         new TThreadPoolServer.Args(serverTransport).processor(processor));
     logger.info("Starting remote interpreter server on port {}", port);
     remoteWorksResponsePool = Collections.synchronizedMap(new HashMap<String, Object>());
-    remoteWorksController = new ZeppelinRemoteWorksController(this, remoteWorksResponsePool);
   }
 
   @Override
   public void run() {
-    if (null != callbackHost && !isTest) {
+    if (null != intpEventServerHost && !isTest) {
       new Thread(new Runnable() {
         boolean interrupted = false;
 
@@ -183,12 +195,11 @@ public class RemoteInterpreterServer extends Thread
           }
 
           if (!interrupted) {
-            CallbackInfo callbackInfo = new CallbackInfo(host, port);
+            RegisterInfo registerInfo = new RegisterInfo(host, port, interpreterGroupId);
             try {
-              RemoteInterpreterUtils
-                  .registerInterpreter(callbackHost, callbackPort, callbackInfo);
+              intpEventServiceClient.registerInterpreterProcess(registerInfo);
             } catch (TException e) {
-              logger.error("Error while registering interpreter: {}", callbackInfo, e);
+              logger.error("Error while registering interpreter: {}", registerInfo, e);
               try {
                 shutdown();
               } catch (TException e1) {
@@ -205,7 +216,6 @@ public class RemoteInterpreterServer extends Thread
   @Override
   public void shutdown() throws TException {
     logger.info("Shutting down...");
-    eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
     if (interpreterGroup != null) {
       for (List<Interpreter> session : interpreterGroup.values()) {
         for (Interpreter interpreter : session) {
@@ -254,21 +264,20 @@ public class RemoteInterpreterServer extends Thread
 
   public static void main(String[] args)
       throws TTransportException, InterruptedException, IOException {
-    Class klass = RemoteInterpreterServer.class;
-    URL location = klass.getResource('/' + klass.getName().replace('.', '/') + ".class");
-    logger.info("URL:" + location);
-    String callbackHost = null;
+    String zeppelinServerHost = null;
     int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT;
     String portRange = ":";
+    String interpreterGroupId = null;
     if (args.length > 0) {
-      callbackHost = args[0];
+      zeppelinServerHost = args[0];
       port = Integer.parseInt(args[1]);
-      if (args.length > 2) {
-        portRange = args[2];
+      interpreterGroupId = args[2];
+      if (args.length > 3) {
+        portRange = args[3];
       }
     }
     RemoteInterpreterServer remoteInterpreterServer =
-        new RemoteInterpreterServer(callbackHost, port, portRange);
+        new RemoteInterpreterServer(zeppelinServerHost, port, interpreterGroupId, portRange);
     remoteInterpreterServer.start();
     remoteInterpreterServer.join();
     System.exit(0);
@@ -279,12 +288,13 @@ public class RemoteInterpreterServer extends Thread
       className, Map<String, String> properties, String userName) throws TException {
     if (interpreterGroup == null) {
       interpreterGroup = new InterpreterGroup(interpreterGroupId);
-      angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this);
+      angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), intpEventClient);
       hookRegistry = new InterpreterHookRegistry();
-      resourcePool = new DistributedResourcePool(interpreterGroup.getId(), eventClient);
+      resourcePool = new DistributedResourcePool(interpreterGroup.getId(), intpEventClient);
       interpreterGroup.setInterpreterHookRegistry(hookRegistry);
       interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
       interpreterGroup.setResourcePool(resourcePool);
+      intpEventClient.setIntpGroupId(interpreterGroupId);
 
       String localRepoPath = properties.get("zeppelin.interpreter.localRepo");
       if (properties.containsKey("zeppelin.interpreter.output.limit")) {
@@ -327,8 +337,8 @@ public class RemoteInterpreterServer extends Thread
     return resourcePool;
   }
 
-  protected RemoteInterpreterEventClient getEventClient() {
-    return eventClient;
+  protected RemoteInterpreterEventClient getIntpEventClient() {
+    return intpEventClient;
   }
 
   private void setSystemProperty(Properties properties) {
@@ -388,7 +398,7 @@ public class RemoteInterpreterServer extends Thread
           logger.info("Unload App {} ", appInfo.pkg.getName());
           appInfo.app.unload();
           // see ApplicationState.Status.UNLOADED
-          eventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED");
+          intpEventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED");
         } catch (ApplicationException e) {
           logger.error(e.getMessage(), e);
         }
@@ -739,32 +749,48 @@ public class RemoteInterpreterServer extends Thread
   }
 
   private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) {
-    List<InterpreterContextRunner> contextRunners = new LinkedList<>();
-    List<InterpreterContextRunner> runners = gson.fromJson(ric.getRunners(),
-        new TypeToken<List<RemoteInterpreterContextRunner>>() {
-        }.getType());
+    //    List<InterpreterContextRunner> contextRunners = new LinkedList<>();
+    //    List<InterpreterContextRunner> runners = gson.fromJson(ric.getRunners(),
+    //        new TypeToken<List<RemoteInterpreterContextRunner>>() {
+    //        }.getType());
+    //
+    //    if (runners != null) {
+    //      for (InterpreterContextRunner r : runners) {
+    //        contextRunners.add(new ParagraphRunner(this, r.getNoteId(), r.getParagraphId()));
+    //      }
+    //    }
 
-    if (runners != null) {
-      for (InterpreterContextRunner r : runners) {
-        contextRunners.add(new ParagraphRunner(this, r.getNoteId(), r.getParagraphId()));
-      }
-    }
-
-    return new InterpreterContext(
-        ric.getNoteId(),
-        ric.getParagraphId(),
-        ric.getReplName(),
-        ric.getParagraphTitle(),
-        ric.getParagraphText(),
-        AuthenticationInfo.fromJson(ric.getAuthenticationInfo()),
-        (Map<String, Object>) gson.fromJson(ric.getConfig(),
-            new TypeToken<Map<String, Object>>() {
-            }.getType()),
-        GUI.fromJson(ric.getGui()),
-        GUI.fromJson(ric.getNoteGui()),
-        interpreterGroup.getAngularObjectRegistry(),
-        interpreterGroup.getResourcePool(),
-        contextRunners, output, remoteWorksController, eventClient, progressMap);
+    return InterpreterContext.builder()
+        .setNoteId(ric.getNoteId())
+        .setParagraphId(ric.getParagraphId())
+        .setReplName(ric.getReplName())
+        .setParagraphTitle(ric.getParagraphTitle())
+        .setParagraphText(ric.getParagraphText())
+        .setAuthenticationInfo(AuthenticationInfo.fromJson(ric.getAuthenticationInfo()))
+        .setGUI(GUI.fromJson(ric.getGui()))
+        .setNoteGUI(GUI.fromJson(ric.getNoteGui()))
+        .setAngularObjectRegistry(interpreterGroup.getAngularObjectRegistry())
+        .setResourcePool(interpreterGroup.getResourcePool())
+        .setInterpreterOut(output)
+        .setIntpEventClient(intpEventClient)
+        .setProgressMap(progressMap)
+        .build();
+    //    return new InterpreterContext(
+    //        ric.getNoteId(),
+    //        ric.getParagraphId(),
+    //        ric.getReplName(),
+    //        ric.getParagraphTitle(),
+    //        ric.getParagraphText(),
+    //        AuthenticationInfo.fromJson(ric.getAuthenticationInfo()),
+    //        (Map<String, Object>) gson.fromJson(ric.getConfig(),
+    //            new TypeToken<Map<String, Object>>() {
+    //            }.getType()),
+    //        GUI.fromJson(ric.getGui()),
+    //        GUI.fromJson(ric.getNoteGui()),
+    //        interpreterGroup.getAngularObjectRegistry(),
+    //        interpreterGroup.getResourcePool(),
+    //        output, intpEventClient,
+    //        progressMap);
   }
 
 
@@ -774,7 +800,7 @@ public class RemoteInterpreterServer extends Thread
       @Override
       public void onUpdateAll(InterpreterOutput out) {
         try {
-          eventClient.onInterpreterOutputUpdateAll(
+          intpEventClient.onInterpreterOutputUpdateAll(
               noteId, paragraphId, out.toInterpreterResultMessage());
         } catch (IOException e) {
           logger.error(e.getMessage(), e);
@@ -785,7 +811,7 @@ public class RemoteInterpreterServer extends Thread
       public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
         String output = new String(line);
         logger.debug("Output Append: {}", output);
-        eventClient.onInterpreterOutputAppend(
+        intpEventClient.onInterpreterOutputAppend(
             noteId, paragraphId, index, output);
       }
 
@@ -795,7 +821,7 @@ public class RemoteInterpreterServer extends Thread
         try {
           output = new String(out.toByteArray());
           logger.debug("Output Update for index {}: {}", index, output);
-          eventClient.onInterpreterOutputUpdate(
+          intpEventClient.onInterpreterOutputUpdate(
               noteId, paragraphId, index, out.getType(), output);
         } catch (IOException e) {
           logger.error(e.getMessage(), e);
@@ -816,83 +842,10 @@ public class RemoteInterpreterServer extends Thread
 
     @Override
     public void run() {
-      server.eventClient.run(this);
+//      server.eventClient.run(this);
     }
   }
 
-  static class ZeppelinRemoteWorksController implements RemoteWorksController {
-    Logger logger = LoggerFactory.getLogger(ZeppelinRemoteWorksController.class);
-
-    private final long DEFAULT_TIMEOUT_VALUE = 300000;
-    private final Map<String, Object> remoteWorksResponsePool;
-    private RemoteInterpreterServer server;
-
-    ZeppelinRemoteWorksController(
-        RemoteInterpreterServer server, Map<String, Object> remoteWorksResponsePool) {
-      this.remoteWorksResponsePool = remoteWorksResponsePool;
-      this.server = server;
-    }
-
-    public String generateOwnerKey() {
-      String hashKeyText = new String("ownerKey" + System.currentTimeMillis());
-      String hashKey = String.valueOf(hashKeyText.hashCode());
-      return hashKey;
-    }
-
-    public boolean waitForEvent(String eventOwnerKey) throws InterruptedException {
-      return waitForEvent(eventOwnerKey, DEFAULT_TIMEOUT_VALUE);
-    }
-
-    public boolean waitForEvent(String eventOwnerKey, long timeout) throws InterruptedException {
-      boolean wasGetData = false;
-      long now = System.currentTimeMillis();
-      long endTime = System.currentTimeMillis() + timeout;
-
-      while (endTime >= now) {
-        synchronized (this.remoteWorksResponsePool) {
-          wasGetData = this.remoteWorksResponsePool.containsKey(eventOwnerKey);
-        }
-        if (wasGetData == true) {
-          break;
-        }
-        now = System.currentTimeMillis();
-        sleep(500);
-      }
-
-      return wasGetData;
-    }
-
-    @Override
-    public List<InterpreterContextRunner> getRemoteContextRunner(String noteId) {
-      return getRemoteContextRunner(noteId, null);
-    }
-
-    public List<InterpreterContextRunner> getRemoteContextRunner(
-        String noteId, String paragraphID) {
-
-      List<InterpreterContextRunner> runners = null;
-      String ownerKey = generateOwnerKey();
-
-      ZeppelinServerResourceParagraphRunner resource = new ZeppelinServerResourceParagraphRunner();
-      resource.setNoteId(noteId);
-      resource.setParagraphId(paragraphID);
-      server.eventClient.getZeppelinServerNoteRunner(ownerKey, resource);
-
-      try {
-        this.waitForEvent(ownerKey);
-      } catch (Exception e) {
-        return new LinkedList<>();
-      }
-      synchronized (this.remoteWorksResponsePool) {
-        runners = (List<InterpreterContextRunner>) this.remoteWorksResponsePool.get(ownerKey);
-        this.remoteWorksResponsePool.remove(ownerKey);
-      }
-      return runners;
-    }
-
-
-  }
-
   private RemoteInterpreterResult convert(InterpreterResult result,
                                           Map<String, Object> config, GUI gui, GUI noteGui) {
 
@@ -921,54 +874,30 @@ public class RemoteInterpreterServer extends Thread
     synchronized (interpreterGroup) {
       List<Interpreter> interpreters = interpreterGroup.get(sessionId);
       if (interpreters == null) {
+        logger.info("getStatus:" + Status.UNKNOWN.name());
         return Status.UNKNOWN.name();
       }
       //TODO(zjffdu) ineffient for loop interpreter and its jobs
       for (Interpreter intp : interpreters) {
         for (Job job : intp.getScheduler().getJobsRunning()) {
           if (jobId.equals(job.getId())) {
+            logger.info("getStatus:" + job.getStatus().name());
             return job.getStatus().name();
           }
         }
 
         for (Job job : intp.getScheduler().getJobsWaiting()) {
           if (jobId.equals(job.getId())) {
+            logger.info("getStatus:" + job.getStatus().name());
             return job.getStatus().name();
           }
         }
       }
     }
+    logger.info("getStatus:" + Status.UNKNOWN.name());
     return Status.UNKNOWN.name();
   }
 
-
-  @Override
-  public void onAdd(String interpreterGroupId, AngularObject object) {
-    eventClient.angularObjectAdd(object);
-  }
-
-  @Override
-  public void onUpdate(String interpreterGroupId, AngularObject object) {
-    eventClient.angularObjectUpdate(object);
-  }
-
-  @Override
-  public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
-    eventClient.angularObjectRemove(name, noteId, paragraphId);
-  }
-
-
-  /**
-   * Poll event from RemoteInterpreterEventPoller
-   *
-   * @return
-   * @throws TException
-   */
-  @Override
-  public RemoteInterpreterEvent getEvent() throws TException {
-    return eventClient.pollEvent();
-  }
-
   /**
    * called when object is updated in client (web) side.
    *
@@ -1068,26 +997,9 @@ public class RemoteInterpreterServer extends Thread
     registry.remove(name, noteId, paragraphId, false);
   }
 
-  @Override
-  public void resourcePoolResponseGetAll(List<String> resources) throws TException {
-    eventClient.putResponseGetAllResources(resources);
-  }
-
-  /**
-   * Get payload of resource from remote
-   *
-   * @param resourceId json serialized ResourceId
-   * @param object     java serialized of the object
-   * @throws TException
-   */
-  @Override
-  public void resourceResponseGet(String resourceId, ByteBuffer object) throws TException {
-    eventClient.putResponseGetResource(resourceId, object);
-  }
-
   @Override
   public List<String> resourcePoolGetAll() throws TException {
-    logger.debug("Request getAll from ZeppelinServer");
+    logger.debug("Request resourcePoolGetAll from ZeppelinServer");
     List<String> result = new LinkedList<>();
 
     if (resourcePool == null) {
@@ -1170,30 +1082,30 @@ public class RemoteInterpreterServer extends Thread
     }
   }
 
-  /**
-   * Get payload of resource from remote
-   *
-   * @param invokeResourceMethodEventMessage json serialized InvokeResourcemethodEventMessage
-   * @param object                           java serialized of the object
-   * @throws TException
-   */
-  @Override
-  public void resourceResponseInvokeMethod(
-      String invokeResourceMethodEventMessage, ByteBuffer object) throws TException {
-    InvokeResourceMethodEventMessage message =
-        InvokeResourceMethodEventMessage.fromJson(invokeResourceMethodEventMessage);
-
-    if (message.shouldPutResultIntoResourcePool()) {
-      Resource resource = resourcePool.get(
-          message.resourceId.getNoteId(),
-          message.resourceId.getParagraphId(),
-          message.returnResourceName,
-          true);
-      eventClient.putResponseInvokeMethod(message, resource);
-    } else {
-      eventClient.putResponseInvokeMethod(message, object);
-    }
-  }
+  //  /**
+  //   * Get payload of resource from remote
+  //   *
+  //   * @param invokeResourceMethodEventMessage json serialized InvokeResourcemethodEventMessage
+  //   * @param object                           java serialized of the object
+  //   * @throws TException
+  //   */
+  //  @Override
+  //  public void resourceResponseInvokeMethod(
+  //      String invokeResourceMethodEventMessage, ByteBuffer object) throws TException {
+  //    InvokeResourceMethodEventMessage message =
+  //        InvokeResourceMethodEventMessage.fromJson(invokeResourceMethodEventMessage);
+  //
+  //    if (message.shouldPutResultIntoResourcePool()) {
+  //      Resource resource = resourcePool.get(
+  //          message.resourceId.getNoteId(),
+  //          message.resourceId.getParagraphId(),
+  //          message.returnResourceName,
+  //          true);
+  //      eventClient.putResponseInvokeMethod(message, resource);
+  //    } else {
+  //      eventClient.putResponseInvokeMethod(message, object);
+  //    }
+  //  }
 
   @Override
   public void angularRegistryPush(String registryAsString) throws TException {
@@ -1219,13 +1131,13 @@ public class RemoteInterpreterServer extends Thread
 
       @Override
       public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
-        eventClient.onAppOutputAppend(noteId, paragraphId, index, appId, new String(line));
+        intpEventClient.onAppOutputAppend(noteId, paragraphId, index, appId, new String(line));
       }
 
       @Override
       public void onUpdate(int index, InterpreterResultMessageOutput out) {
         try {
-          eventClient.onAppOutputUpdate(noteId, paragraphId, index, appId,
+          intpEventClient.onAppOutputUpdate(noteId, paragraphId, index, appId,
               out.getType(), new String(out.toByteArray()));
         } catch (IOException e) {
           logger.error(e.getMessage(), e);
@@ -1318,7 +1230,7 @@ public class RemoteInterpreterServer extends Thread
         runningApp.app.run(resource);
         context.out.flush();
         InterpreterResultMessageOutput out = context.out.getOutputAt(0);
-        eventClient.onAppOutputUpdate(
+        intpEventClient.onAppOutputUpdate(
             context.getNoteId(),
             context.getParagraphId(),
             0,