Merge remote-tracking branch 'origin/master' into ZEPPELIN-3001

Change-Id: I0be128108b3b80643162aa411b9b1a158badcbb1

.travis.yml

@@ -44,7 +44,7 @@ matrix:
     # Test License compliance using RAT tool
     - jdk: "openjdk7"
       dist: trusty
-      env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
+      env: BUILD_PLUGINS="false" SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
 
     # Run e2e tests (in zeppelin-web)
     # chrome dropped the support for precise (ubuntu 12.04), so need to use trusty
@@ -53,7 +53,7 @@ matrix:
       sudo: false
       dist: trusty
       jdk: "oraclejdk8"
-      env: CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
+      env: BUILD_PLUGINS="false" CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
       addons:
         apt:
           packages:
@@ -66,19 +66,19 @@ matrix:
     - sudo: required
       jdk: "oraclejdk8"
       dist: trusty
-      env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
 
     # Test selenium with spark module for 1.6.3
     - jdk: "oraclejdk8"
       dist: trusty
       addons:
         firefox: "31.0"
-      env: CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false"
 
     # Test interpreter modules
     - jdk: "openjdk7"
       dist: trusty
-      env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+      env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
 
     # Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately
     
@@ -86,35 +86,35 @@ matrix:
     - sudo: required
       jdk: "oraclejdk8"
       dist: trusty
-      env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Integration test of spark interpreter with different spark versions under python3, only run SparkIntegrationTest. Also run spark unit test of spark 1.6 in this build.
     - sudo: required
       jdk: "oraclejdk8"
       dist: trusty
-      env: PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 2.1.0 with scala 2.11
     - jdk: "openjdk7"
       dist: trusty
-      env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test spark module for 2.0.2 with scala 2.11
     - jdk: "oraclejdk8"
       dist: trusty
-      env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
     # Test python/pyspark with python 2, livy 0.5
     - sudo: required
       dist: trusty
       jdk: "openjdk7"
-      env: PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
+      env: BUILD_PLUGINS="false" PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
 
     # Test livy 0.5 with spark 2.2.0 under python3
     - sudo: required
       dist: trusty
       jdk: "openjdk8"
-      env: PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
+      env: BUILD_PLUGINS="false" PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
 
 before_install:
   # check files included in commit range, clear bower_components if a bower.json file has changed.
@@ -136,6 +136,7 @@ before_install:
 install:
   - echo "mvn $BUILD_FLAG $MODULES $PROFILE -B"
   - mvn $BUILD_FLAG $MODULES $PROFILE -B
+  - if [ $BUILD_PLUGINS == "true" ]; then echo "mvn clean package -pl zeppelin-plugins -amd -B"; mvn clean package -pl zeppelin-plugins -amd -B; fi
 
 before_script:
   - if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi

docs/interpreter/shell.md

@@ -35,7 +35,7 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
 <table class="table-configuration">
   <tr>
     <th>Name</th>
-    <th>Value</th>
+    <th>Default</th>
     <th>Description</th>
   </tr>
   <tr>
@@ -63,6 +63,11 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
     <td></td>
     <td>The path to the keytab file</td>
   </tr>
+  <tr>
+    <td>zeppelin.shell.interpolation</td>
+    <td>false</td>
+    <td>Enable ZeppelinContext variable interpolation into paragraph text</td>
+  </tr>
 </table>
 
 ## Example
@@ -82,3 +87,25 @@ export LAUNCH_KERBEROS_REFRESH_INTERVAL=4h
 # Change kinit number retries (default value is 5), which means if the kinit command fails for 5 retries consecutively it will close the interpreter. 
 export KINIT_FAIL_THRESHOLD=10
 ```
+
+## Object Interpolation
+The shell interpreter also supports interpolation of `ZeppelinContext` objects into the paragraph text.
+The following example shows one use of this facility:
+
+####In Scala cell:
+```
+z.put("dataFileName", "members-list-003.parquet")
+    // ...
+val members = spark.read.parquet(z.get("dataFileName"))
+    // ...
+```
+
+####In later Shell cell:
+```
+%sh rm -rf {dataFileName}
+```
+
+Object interpolation is disabled by default, and can be enabled (for the Shell interpreter) by 
+setting the value of the property `zeppelin.shell.interpolation` to `true` (see _Configuration_ above). 
+More details of this feature can be found in the Spark interpreter documentation under 
+[Object Interpolation](spark.html#object-interpolation)

docs/interpreter/spark.md

@@ -145,6 +145,11 @@ You can also set other Spark properties which are not listed in the table. For a
     <td>true</td>
     <td>Do not change - developer only setting, not for production use</td>
   </tr>
+  <tr>
+    <td>zeppelin.spark.sql.interpolation</td>
+    <td>false</td>
+    <td>Enable ZeppelinContext variable interpolation into paragraph text</td>
+  </tr>
   <tr>
   <td>zeppelin.spark.uiWebUrl</td>
     <td></td>
@@ -365,6 +370,55 @@ myScalaDataFrame = DataFrame(z.get("myScalaDataFrame"), sqlContext)
   </div>
 </div>
 
+### Object Interpolation
+Some interpreters can interpolate object values from `z` into the paragraph text by using the 
+`{variable-name}` syntax. The value of any object previously `put` into `z` can be 
+interpolated into a paragraph text by using such a pattern containing the object's name. 
+The following example shows one use of this facility:
+
+####In Scala cell:
+```
+z.put("minAge", 35)
+```
+
+####In later SQL cell:
+```
+%sql select * from members where age >= {minAge}
+```
+
+The interpolation of a `{var-name}` pattern is performed only when `z` contains an object with the specified name.
+But the pattern is left unchanged if the named object does not exist in `z`.
+Further, all `{var-name}` patterns within the paragraph text must must be translatable for any interpolation to occur -- 
+translation of only some of the patterns in a paragraph text is never done.
+
+In some situations, it is necessary to use { and } characters in a paragraph text without invoking the 
+object interpolation mechanism. For these cases an escaping mechanism is available -- 
+doubled braces {{ and }} should be used. The following example shows the use of {{ and }} for passing a 
+regular expression containing just { and } into the paragraph text.
+
+```
+%sql select * from members where name rlike '[aeiou]{{3}}'
+```
+
+To summarize, patterns of the form `{var-name}` within the paragraph text will be interpolated only if a predefined 
+object of the specified name exists. Additionally, all such patterns within the paragraph text should also 
+be translatable for any interpolation to occur. Patterns of the form `{{any-text}}` are translated into `{any-text}`. 
+These translations are performed only when all occurrences of `{`, `}`, `{{`, and `}}` in the paragraph text conform 
+to one of the two forms described above. Paragraph text containing `{` and/or `}` characters used in any other way 
+(than `{var-name}` and `{{any-text}}`) is used as-is without any changes. 
+No error is flagged in any case. This behavior is identical to the implementation of a similar feature in 
+Jupyter's shell invocation using the `!` magic command.
+
+This feature is disabled by default, and must be explicitly turned on for each interpreter independently 
+by setting the value of an interpreter-specific property to `true`. 
+Consult the _Configuration_ section of each interpreter's documentation 
+to find out if object interpolation is implemented, and the name of the parameter that must be set to `true` to 
+enable the feature. The name of the parameter used to enable this feature it is different for each interpreter. 
+For example, the SparkSQL and Shell interpreters use the parameter names `zeppelin.spark.sql.interpolation` and 
+`zeppelin.shell.interpolation` respectively.
+
+At present only the SparkSQL and Shell interpreters support object interpolation. 
+
 ### Form Creation
 
 `ZeppelinContext` provides functions for creating forms.

kylin/pom.xml

@@ -73,7 +73,13 @@
             <plugin>
                 <artifactId>maven-resources-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
-
-</project>
+</project>

kylin/src/main/java/org/apache/zeppelin/kylin/KylinErrorResponse.java

@@ -34,7 +34,7 @@ class KylinErrorResponse implements JsonSerializable {
   private Object data;
   private String msg;
 
-  public KylinErrorResponse(String stacktrace, String exception, String url,
+  KylinErrorResponse(String stacktrace, String exception, String url,
       String code, Object data, String msg) {
     this.stacktrace = stacktrace;
     this.exception = exception;
@@ -59,5 +59,4 @@ class KylinErrorResponse implements JsonSerializable {
       return null;
     }
   }
-
 }

kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java

@@ -24,10 +24,6 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,6 +33,11 @@ import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+
 /**
  * Kylin interpreter for Zeppelin. (http://kylin.apache.org)
  */
@@ -233,5 +234,4 @@ public class KylinInterpreter extends Interpreter {
     }
     return res.toString();
   }
-
 }

kylin/src/test/java/org/apache/zeppelin/kylin/KylinInterpreterTest.java

@@ -19,13 +19,6 @@ package org.apache.zeppelin.kylin;
 
 import static org.junit.Assert.assertEquals;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Locale;
-import java.util.Properties;
-
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -33,23 +26,31 @@ import org.apache.http.ProtocolVersion;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.message.AbstractHttpMessage;
-import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Locale;
+import java.util.Properties;
+
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
 public class KylinInterpreterTest {
-  static final Properties kylinProperties = new Properties();
+  static final Properties KYLIN_PROPERTIES = new Properties();
 
   @BeforeClass
   public static void setUpClass() {
-    kylinProperties.put("kylin.api.url", "http://localhost:7070/kylin/api/query");
-    kylinProperties.put("kylin.api.user", "ADMIN");
-    kylinProperties.put("kylin.api.password", "KYLIN");
-    kylinProperties.put("kylin.query.project", "default");
-    kylinProperties.put("kylin.query.offset", "0");
-    kylinProperties.put("kylin.query.limit", "5000");
-    kylinProperties.put("kylin.query.ispartial", "true");
+    KYLIN_PROPERTIES.put("kylin.api.url", "http://localhost:7070/kylin/api/query");
+    KYLIN_PROPERTIES.put("kylin.api.user", "ADMIN");
+    KYLIN_PROPERTIES.put("kylin.api.password", "KYLIN");
+    KYLIN_PROPERTIES.put("kylin.query.project", "default");
+    KYLIN_PROPERTIES.put("kylin.query.offset", "0");
+    KYLIN_PROPERTIES.put("kylin.query.limit", "5000");
+    KYLIN_PROPERTIES.put("kylin.query.ispartial", "true");
   }
 
   @Test
@@ -58,47 +59,57 @@ public class KylinInterpreterTest {
     InterpreterResult result = t.interpret(
         "select a.date,sum(b.measure) as measure from kylin_fact_table a " +
             "inner join kylin_lookup_table b on a.date=b.date group by a.date", null);
-    assertEquals("default", t.getProject("select a.date,sum(b.measure) as measure " +
-                    "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
-    assertEquals(InterpreterResult.Type.TABLE,result.message().get(0).getType());
+    assertEquals("default", t.getProject("select a.date,sum(b.measure) as measure "
+            + "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+            + "group by a.date"));
+    assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
   }
 
   @Test
   public void testWithProject() {
     KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
-    assertEquals("project2", t.getProject("(project2)\n select a.date,sum(b.measure) as measure " +
-            "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
-    assertEquals("", t.getProject("()\n select a.date,sum(b.measure) as measure " +
-            "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
-    assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a inner join " +
-            "kylin_lookup_table b on a.date=b.date group by a.date", t.getSQL("(project2)\n select a.date," +
-            "sum(b.measure) as measure from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date " +
-            "group by a.date"));
-    assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a inner join kylin_lookup_table b " +
-            "on a.date=b.date group by a.date", t.getSQL("()\n select a.date,sum(b.measure) as measure " +
-            "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
+    assertEquals("project2", t.getProject("(project2)\n select a.date,sum(b.measure) "
+            + "as measure from kylin_fact_table a inner join kylin_lookup_table b on "
+            + "a.date=b.date group by a.date"));
+    assertEquals("", t.getProject("()\n select a.date,sum(b.measure) as measure "
+            + "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+            + "group by a.date"));
+    assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+            + "inner join kylin_lookup_table b on a.date=b.date group by a.date",
+            t.getSQL("(project2)\n select a.date,sum(b.measure) as measure "
+                    + "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+                    + "group by a.date"));
+    assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+            + "inner join kylin_lookup_table b on a.date=b.date group by a.date",
+            t.getSQL("()\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+                    + "inner join kylin_lookup_table b on a.date=b.date group by a.date"));
   }
 
   @Test
   public void testParseResult() {
-    String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\",\"name\":\"COUNTRY\","
-            + "\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\",\"precision\":256,"
-            + "\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
-            + "\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,"
-            + "\"currency\":false,\"signed\":true},{\"isNullable\":1,\"displaySize\":256,\"label\":\"CURRENCY\","
-            + "\"name\":\"CURRENCY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\","
-            + "\"precision\":256,\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\",\"writable\":false,"
-            + "\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,"
-            + "\"searchable\":false,\"currency\":false,\"signed\":true},{\"isNullable\":0,\"displaySize\":19,"
-            + "\"label\":\"COUNT__\",\"name\":\"COUNT__\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
-            + "\"tableName\":\"SALES_TABLE\",\"precision\":19,\"scale\":0,\"columnType\":-5,\"columnTypeName\":"
-            + "\"BIGINT\",\"writable\":false,\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,"
-            + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"signed\":true}],\"results\":"
-            + "[[\"AMERICA\",\"USD\",null],[null,\"RMB\",0],[\"KOR\",null,100],[\"\\\"abc\\\"\",\"a,b,c\",-1]],"
-            + "\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,\"isException\":false,\"exceptionMessage\":null,"
-            + "\"duration\":134,\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
+    String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
+            + "\"name\":\"COUNTRY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
+            + "\"tableName\":\"SALES_TABLE\",\"precision\":256,\"scale\":0,\"columnType\":12,"
+            + "\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
+            + "\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,"
+            + "\"searchable\":false,\"currency\":false,\"signed\":true},{\"isNullable\":1,"
+            + "\"displaySize\":256,\"label\":\"CURRENCY\",\"name\":\"CURRENCY\","
+            + "\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\","
+            + "\"precision\":256,\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\","
+            + "\"writable\":false,\"readOnly\":true,\"definitelyWritable\":false,"
+            + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,"
+            + "\"currency\":false,\"signed\":true},{\"isNullable\":0,\"displaySize\":19,"
+            + "\"label\":\"COUNT__\",\"name\":\"COUNT__\",\"schemaName\":\"DEFAULT\","
+            + "\"catelogName\":null,\"tableName\":\"SALES_TABLE\",\"precision\":19,\"scale\":0,"
+            + "\"columnType\":-5,\"columnTypeName\":\"BIGINT\",\"writable\":false,"
+            + "\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,"
+            + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"signed\":true}],"
+            + "\"results\":[[\"AMERICA\",\"USD\",null],[null,\"RMB\",0],[\"KOR\",null,100],"
+            + "[\"\\\"abc\\\"\",\"a,b,c\",-1]],\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,"
+            + "\"isException\":false,\"exceptionMessage\":null,\"duration\":134,"
+            + "\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
             + "\"partial\":false}";
-    String expected="%table COUNTRY \tCURRENCY \tCOUNT__ \t \n" +
+    String expected = "%table COUNTRY \tCURRENCY \tCOUNT__ \t \n" +
             "AMERICA \tUSD \tnull \t \n" +
             "null \tRMB \t0 \t \n" +
             "KOR \tnull \t100 \t \n" +
@@ -110,23 +121,28 @@ public class KylinInterpreterTest {
 
   @Test
   public void testParseEmptyResult() {
-    String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\",\"name\":\"COUNTRY\","
-        + "\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\",\"precision\":256,"
-        + "\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
-        + "\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,"
-        + "\"currency\":false,\"signed\":true},{\"isNullable\":1,\"displaySize\":256,\"label\":\"CURRENCY\","
-        + "\"name\":\"CURRENCY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\","
-        + "\"precision\":256,\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\",\"writable\":false,"
-        + "\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,"
-        + "\"searchable\":false,\"currency\":false,\"signed\":true},{\"isNullable\":0,\"displaySize\":19,"
-        + "\"label\":\"COUNT__\",\"name\":\"COUNT__\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
-        + "\"tableName\":\"SALES_TABLE\",\"precision\":19,\"scale\":0,\"columnType\":-5,\"columnTypeName\":"
-        + "\"BIGINT\",\"writable\":false,\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,"
-        + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"signed\":true}],\"results\":"
-        + "[]," + "\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,\"isException\":false,\"exceptionMessage\":null,"
-        + "\"duration\":134,\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
-        + "\"partial\":false}";
-    String expected="%table COUNTRY \tCURRENCY \tCOUNT__ \t \n";
+    String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
+            + "\"name\":\"COUNTRY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
+            + "\"tableName\":\"SALES_TABLE\",\"precision\":256,\"scale\":0,\"columnType\":12,"
+            + "\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
+            + "\"definitelyWritable\":false,\"autoIncrement\":false,\"caseSensitive\":true,"
+            + "\"searchable\":false,\"currency\":false,\"signed\":true},{\"isNullable\":1,"
+            + "\"displaySize\":256,\"label\":\"CURRENCY\",\"name\":\"CURRENCY\","
+            + "\"schemaName\":\"DEFAULT\",\"catelogName\":null,\"tableName\":\"SALES_TABLE\","
+            + "\"precision\":256,\"scale\":0,\"columnType\":12,\"columnTypeName\":\"VARCHAR\","
+            + "\"writable\":false,\"readOnly\":true,\"definitelyWritable\":false,"
+            + "\"autoIncrement\":false,\"caseSensitive\":true,\"searchable\":false,"
+            + "\"currency\":false,\"signed\":true},{\"isNullable\":0,\"displaySize\":19,"
+            + "\"label\":\"COUNT__\",\"name\":\"COUNT__\",\"schemaName\":\"DEFAULT\","
+            + "\"catelogName\":null,\"tableName\":\"SALES_TABLE\",\"precision\":19,\"scale\":0,"
+            + "\"columnType\":-5,\"columnTypeName\":\"BIGINT\",\"writable\":false,"
+            + "\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,"
+            + "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"signed\":true}],"
+            + "\"results\":[]," + "\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,"
+            + "\"isException\":false,\"exceptionMessage\":null,\"duration\":134,"
+            + "\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
+            + "\"partial\":false}";
+    String expected = "%table COUNTRY \tCURRENCY \tCOUNT__ \t \n";
     KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
     String actual = t.formatResult(msg);
     Assert.assertEquals(expected, actual);
@@ -146,8 +162,7 @@ public class KylinInterpreterTest {
 }
 
 class MockKylinInterpreter extends KylinInterpreter {
-
-  public MockKylinInterpreter(Properties property) {
+  MockKylinInterpreter(Properties property) {
     super(property);
   }
 
@@ -156,7 +171,6 @@ class MockKylinInterpreter extends KylinInterpreter {
     MockHttpClient client = new MockHttpClient();
     return client.execute(new HttpPost());
   }
-
 }
 
 class MockHttpClient{
@@ -166,7 +180,6 @@ class MockHttpClient{
 }
 
 class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
-
   @Override
   public StatusLine getStatusLine() {
     return new MockStatusLine();
@@ -174,27 +187,22 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
 
   @Override
   public void setStatusLine(StatusLine statusLine) {
-
   }
 
   @Override
   public void setStatusLine(ProtocolVersion protocolVersion, int i) {
-
   }
 
   @Override
   public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {
-
   }
 
   @Override
   public void setStatusCode(int i) throws IllegalStateException {
-
   }
 
   @Override
   public void setReasonPhrase(String s) throws IllegalStateException {
-
   }
 
   @Override
@@ -204,7 +212,6 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
 
   @Override
   public void setEntity(HttpEntity httpEntity) {
-
   }
 
   @Override
@@ -214,7 +221,6 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
 
   @Override
   public void setLocale(Locale locale) {
-
   }
 
   @Override
@@ -224,7 +230,6 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
 }
 
 class MockStatusLine implements StatusLine{
-
   @Override
   public ProtocolVersion getProtocolVersion() {
     return null;
@@ -242,7 +247,6 @@ class MockStatusLine implements StatusLine{
 }
 
 class MockEntity implements HttpEntity{
-
   @Override
   public boolean isRepeatable() {
     return false;
@@ -278,7 +282,6 @@ class MockEntity implements HttpEntity{
 
   @Override
   public void writeTo(OutputStream outputStream) throws IOException {
-
   }
 
   @Override
@@ -288,6 +291,5 @@ class MockEntity implements HttpEntity{
 
   @Override
   public void consumeContent() throws IOException {
-
   }
 }

livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java

@@ -662,11 +662,12 @@ public abstract class BaseLivyInterpreter extends Interpreter {
       } else {
         restTemplate = new KerberosRestTemplate(keytabLocation, principal, httpClient);
       }
-    }
-    if (httpClient == null) {
-      restTemplate = new RestTemplate();
     } else {
-      restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient));
+      if (httpClient == null) {
+        restTemplate = new RestTemplate();
+      } else {
+        restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient));
+      }
     }
     restTemplate.getMessageConverters().add(0, new StringHttpMessageConverter(Charset.forName("UTF-8")));
     return restTemplate;

markdown/pom.xml

@@ -93,7 +93,13 @@
       <plugin>
         <artifactId>maven-resources-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <configuration>
+          <skip>false</skip>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
-
-</project>
+</project>

markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java

@@ -17,21 +17,20 @@
 
 package org.apache.zeppelin.markdown;
 
-import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.List;
 import java.util.Properties;
 
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterUtils;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * MarkdownInterpreter interpreter for Zeppelin.
@@ -74,7 +73,7 @@ public class Markdown extends Interpreter {
     if (MarkdownParserType.PEGDOWN.toString().equals(parserType)) {
       return new PegdownParser();
     } else {
-      /** default parser. */
+      // default parser
       return new Markdown4jParser();
     }
   }

markdown/src/main/java/org/apache/zeppelin/markdown/Markdown4jParser.java

@@ -18,8 +18,6 @@
 package org.apache.zeppelin.markdown;
 
 import org.markdown4j.Markdown4jProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 

markdown/src/main/java/org/apache/zeppelin/markdown/PegdownWebSequencelPlugin.java

@@ -38,10 +38,9 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 
 /**
- * Pegdown plugin for Websequence diagram
+ * Pegdown plugin for Websequence diagram.
  */
 public class PegdownWebSequencelPlugin extends Parser implements BlockPluginParser {
-
   private static final String WEBSEQ_URL = "http://www.websequencediagrams.com";
 
   public PegdownWebSequencelPlugin() {
@@ -57,29 +56,29 @@ public class PegdownWebSequencelPlugin extends Parser implements BlockPluginPars
 
   public static final String TAG = "%%%";
 
-  Rule StartMarker() {
+  Rule startMarker() {
     return Sequence(Spn1(), TAG, Sp(), "sequence", Sp());
   }
 
-  String EndMarker() {
+  String endMarker() {
     return TAG;
   }
 
-  Rule Body() {
+  Rule body() {
     return OneOrMore(TestNot(TAG), BaseParser.ANY);
   }
 
-  Rule BlockRule() {
+  Rule blockRule() {
     StringBuilderVar style = new StringBuilderVar();
     StringBuilderVar body = new StringBuilderVar();
 
     return NodeSequence(
-        StartMarker(),
+        startMarker(),
         Optional(
             String("style="),
             Sequence(OneOrMore(Letter()), style.append(match()), Spn1())),
-        Sequence(Body(), body.append(match())),
-        EndMarker(),
+        Sequence(body(), body.append(match())),
+        endMarker(),
         push(
             new ExpImageNode("title",
                 createWebsequenceUrl(style.getString(), body.getString()),
@@ -87,9 +86,7 @@ public class PegdownWebSequencelPlugin extends Parser implements BlockPluginPars
     );
   }
 
-  public static String createWebsequenceUrl(String style,
-                                            String content) {
-
+  public static String createWebsequenceUrl(String style, String content) {
     style = StringUtils.defaultString(style, "default");
 
     OutputStreamWriter writer = null;
@@ -144,6 +141,6 @@ public class PegdownWebSequencelPlugin extends Parser implements BlockPluginPars
 
   @Override
   public Rule[] blockPluginRules() {
-    return new Rule[]{BlockRule()};
+    return new Rule[]{blockRule()};
   }
 }

markdown/src/main/java/org/apache/zeppelin/markdown/PegdownYumlPlugin.java

@@ -33,10 +33,9 @@ import java.net.URLEncoder;
 import java.util.Map;
 
 /**
- * Pegdown plugin for YUML
+ * Pegdown plugin for YUML.
  */
 public class PegdownYumlPlugin extends Parser implements BlockPluginParser {
-
   public PegdownYumlPlugin() {
     super(PegdownParser.OPTIONS,
         PegdownParser.PARSING_TIMEOUT_AS_MILLIS,
@@ -52,41 +51,41 @@ public class PegdownYumlPlugin extends Parser implements BlockPluginParser {
 
   public static final String TAG = "%%%";
 
-  Rule StartMarker() {
+  Rule startMarker() {
     return Sequence(Spn1(), TAG, Sp(), "yuml", Sp());
   }
 
-  String EndMarker() {
+  String endMarker() {
     return TAG;
   }
 
-  Rule ParameterName() {
+  Rule parameterName() {
     return FirstOf("type", "style", "scale", "format", "dir");
   }
 
-  Rule Body() {
+  Rule body() {
     return OneOrMore(TestNot(TAG), BaseParser.ANY);
   }
 
-  Rule BlockRule() {
-    ParamVar<String, String> params = new ParamVar<String, String>();
+  Rule blockRule() {
+    ParamVar<String, String> params = new ParamVar<>();
     StringBuilderVar name = new StringBuilderVar();
     StringBuilderVar value = new StringBuilderVar();
     StringBuilderVar body = new StringBuilderVar();
 
     return NodeSequence(
-        StartMarker(),
+        startMarker(),
         ZeroOrMore(
             Sequence(
-                ParameterName(), name.append(match()),
+                parameterName(), name.append(match()),
                 String("="),
                 OneOrMore(Alphanumeric()), value.append(match())),
             Sp(),
             params.put(name.getString(), value.getString()),
             name.clear(), value.clear()),
-        Body(),
+        body(),
         body.append(match()),
-        EndMarker(),
+        endMarker(),
         push(
             new ExpImageNode(
                 "title", createYumlUrl(params.get(), body.getString()), new TextNode("")))
@@ -138,6 +137,6 @@ public class PegdownYumlPlugin extends Parser implements BlockPluginParser {
 
   @Override
   public Rule[] blockPluginRules() {
-    return new Rule[]{BlockRule()};
+    return new Rule[]{blockRule()};
   }
 }

markdown/src/test/java/org/apache/zeppelin/markdown/Markdown4jParserTest.java

@@ -17,21 +17,21 @@
 
 package org.apache.zeppelin.markdown;
 
-import org.apache.zeppelin.interpreter.InterpreterResult;
+import static org.junit.Assert.assertEquals;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Properties;
 
-import static org.junit.Assert.assertEquals;
+import org.apache.zeppelin.interpreter.InterpreterResult;
 
 public class Markdown4jParserTest {
-
   Markdown md;
 
   @Before
-  public void setUp() throws Exception {
+  public void setUp() {
     Properties props = new Properties();
     props.put(Markdown.MARKDOWN_PARSER_TYPE, Markdown.PARSER_TYPE_MARKDOWN4J);
     md = new Markdown(props);
@@ -39,7 +39,7 @@ public class Markdown4jParserTest {
   }
 
   @After
-  public void tearDown() throws Exception {
+  public void tearDown() {
     md.close();
   }
 

markdown/src/test/java/org/apache/zeppelin/markdown/PegdownParserTest.java

@@ -18,20 +18,24 @@
 package org.apache.zeppelin.markdown;
 
 import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Properties;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-
-import static org.apache.zeppelin.markdown.PegdownParser.wrapWithMarkdownClassDiv;
 import static org.junit.Assert.assertThat;
 
+import static org.apache.zeppelin.markdown.PegdownParser.wrapWithMarkdownClassDiv;
+
 import org.hamcrest.CoreMatchers;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
 import org.junit.rules.ErrorCollector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
+import java.util.Properties;
+
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
 public class PegdownParserTest {
   Logger logger = LoggerFactory.getLogger(PegdownParserTest.class);
   Markdown md;
@@ -40,7 +44,7 @@ public class PegdownParserTest {
   public ErrorCollector collector = new ErrorCollector();
 
   @Before
-  public void setUp() throws Exception {
+  public void setUp() {
     Properties props = new Properties();
     props.put(Markdown.MARKDOWN_PARSER_TYPE, Markdown.PARSER_TYPE_PEGDOWN);
     md = new Markdown(props);
@@ -48,13 +52,13 @@ public class PegdownParserTest {
   }
 
   @After
-  public void tearDown() throws Exception {
+  public void tearDown() {
     md.close();
   }
 
   @Test
   public void testMultipleThread() {
-    ArrayList<Thread> arrThreads = new ArrayList<Thread>();
+    ArrayList<Thread> arrThreads = new ArrayList<>();
     for (int i = 0; i < 10; i++) {
       Thread t = new Thread() {
         public void run() {
@@ -112,14 +116,16 @@ public class PegdownParserTest {
   public void testStrikethrough() {
     InterpreterResult result = md.interpret("This is ~~deleted~~ text", null);
     assertEquals(
-        wrapWithMarkdownClassDiv("<p>This is <del>deleted</del> text</p>"), result.message().get(0).getData());
+        wrapWithMarkdownClassDiv("<p>This is <del>deleted</del> text</p>"),
+            result.message().get(0).getData());
   }
 
   @Test
   public void testItalics() {
     InterpreterResult result = md.interpret("This is *italics* text", null);
     assertEquals(
-        wrapWithMarkdownClassDiv("<p>This is <em>italics</em> text</p>"), result.message().get(0).getData());
+        wrapWithMarkdownClassDiv("<p>This is <em>italics</em> text</p>"),
+            result.message().get(0).getData());
   }
 
   @Test
@@ -179,7 +185,8 @@ public class PegdownParserTest {
             .append("[I'm an inline-style link](https://www.google.com)\n")
             .append("\n")
             .append(
-                "[I'm an inline-style link with title](https://www.google.com \"Google's Homepage\")\n")
+                "[I'm an inline-style link with title](https://www.google.com "
+                        + "\"Google's Homepage\")\n")
             .append("\n")
             .append("[I'm a reference-style link][Arbitrary case-insensitive reference text]\n")
             .append("\n")
@@ -205,17 +212,24 @@ public class PegdownParserTest {
             .append(
                 "<p><a href=\"https://www.google.com\">I&rsquo;m an inline-style link</a></p>\n")
             .append(
-                "<p><a href=\"https://www.google.com\" title=\"Google&#39;s Homepage\">I&rsquo;m an inline-style link with title</a></p>\n")
+                "<p><a href=\"https://www.google.com\" title=\"Google&#39;s Homepage\">I&rsquo;m "
+                        + "an inline-style link with title</a></p>\n")
             .append(
                 "<p><a href=\"https://www.mozilla.org\">I&rsquo;m a reference-style link</a></p>\n")
             .append(
-                "<p><a href=\"../blob/master/LICENSE\">I&rsquo;m a relative reference to a repository file</a></p>\n")
+                "<p><a href=\"../blob/master/LICENSE\">I&rsquo;m a relative reference to a "
+                        + "repository file</a></p>\n")
             .append(
-                "<p><a href=\"http://slashdot.org\">You can use numbers for reference-style link definitions</a></p>\n")
+                "<p><a href=\"http://slashdot.org\">You can use numbers for reference-style link "
+                        + "definitions</a></p>\n")
             .append(
-                "<p>Or leave it empty and use the <a href=\"http://www.reddit.com\">link text itself</a>.</p>\n")
+                "<p>Or leave it empty and use the <a href=\"http://www.reddit.com\">link text "
+                        + "itself</a>.</p>\n")
             .append(
-                "<p>URLs and URLs in angle brackets will automatically get turned into links.<br/><a href=\"http://www.example.com\">http://www.example.com</a> or <a href=\"http://www.example.com\">http://www.example.com</a> and sometimes<br/>example.com (but not on Github, for example).</p>\n")
+                "<p>URLs and URLs in angle brackets will automatically get turned into links."
+                        + "<br/><a href=\"http://www.example.com\">http://www.example.com</a> or "
+                        + "<a href=\"http://www.example.com\">http://www.example.com</a> and "
+                        + "sometimes<br/>example.com (but not on Github, for example).</p>\n")
             .append("<p>Some text to show that the reference links can follow later.</p>")
             .toString();
 
@@ -242,19 +256,26 @@ public class PegdownParserTest {
     assertEquals(
         wrapWithMarkdownClassDiv(
             "<blockquote>\n"
-                + "  <p>Blockquotes are very handy in email to emulate reply text.<br/>This line is part of the same quote.</p>\n"
-                + "</blockquote>"),
+                    + "  <p>Blockquotes are very handy in email to emulate reply text.<br/>This "
+                    + "line is part of the same quote.</p>\n"
+                    + "</blockquote>"),
         r1.message().get(0).getData());
 
     InterpreterResult r2 =
         md.interpret(
-            "> This is a very long line that will still be quoted properly when it wraps. Oh boy let's keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can *put* **MarkdownInterpreter** into a blockquote. ",
+            "> This is a very long line that will still be quoted properly when it "
+                    + "wraps. Oh boy let's keep writing to make sure this is long enough to "
+                    + "actually wrap for everyone. Oh, you can *put* **MarkdownInterpreter** "
+                    + "into a blockquote. ",
             null);
     assertEquals(
         wrapWithMarkdownClassDiv(
             "<blockquote>\n"
-                + "  <p>This is a very long line that will still be quoted properly when it wraps. Oh boy let&rsquo;s keep writing to make sure this is long enough to actually wrap for everyone. Oh, you can <em>put</em> <strong>MarkdownInterpreter</strong> into a blockquote. </p>\n"
-                + "</blockquote>"),
+                    + "  <p>This is a very long line that will still be quoted properly when "
+                    + "it wraps. Oh boy let&rsquo;s keep writing to make sure this is long enough "
+                    + "to actually wrap for everyone. Oh, you can <em>put</em> "
+                    + "<strong>MarkdownInterpreter</strong> into a blockquote. </p>\n"
+                    + "</blockquote>"),
         r2.message().get(0).getData());
   }
 
@@ -354,7 +375,8 @@ public class PegdownParserTest {
     // To make unittest independent from websequence service,
     // catch exception, log and pass instead of assert.
     //
-    //assertThat(result.message().get(0).getData(), CoreMatchers.containsString("<img src=\"http://www.websequencediagrams.com/?png="));
+    // assertThat(result.message().get(0).getData(),
+    // CoreMatchers.containsString("<img src=\"http://www.websequencediagrams.com/?png="));
 
     System.err.println(result.message().get(0).getData());
     if (!result.message().get(0).getData().contains(
@@ -375,6 +397,7 @@ public class PegdownParserTest {
         .toString();
 
     InterpreterResult result = md.interpret(input, null);
-    assertThat(result.message().get(0).getData(), CoreMatchers.containsString("<img src=\"http://yuml.me/diagram/"));
+    assertThat(result.message().get(0).getData(),
+            CoreMatchers.containsString("<img src=\"http://yuml.me/diagram/"));
   }
 }

pom.xml

@@ -81,6 +81,7 @@
     <module>zeppelin-web</module>
     <module>zeppelin-server</module>
     <module>zeppelin-jupyter</module>
+    <module>zeppelin-plugins</module>
     <module>zeppelin-distribution</module>
   </modules>
 
@@ -115,6 +116,7 @@
     <commons.logging.version>1.1.1</commons.logging.version>
     <commons.cli.version>1.3.1</commons.cli.version>
     <shiro.version>1.3.2</shiro.version>
+    <joda.version>2.9.9</joda.version>
 
     <!-- test library versions -->
     <junit.version>4.12</junit.version>
@@ -247,6 +249,12 @@
         <version>${commons.cli.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>joda-time</groupId>
+        <artifactId>joda-time</artifactId>
+        <version>${joda.version}</version>
+      </dependency>
+
       <!-- Apache Shiro -->
       <dependency>
         <groupId>org.apache.shiro</groupId>
@@ -592,6 +600,9 @@
           <version>${plugin.surefire.version}</version>
           <configuration combine.children="append">
             <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
+            <environmentVariables>
+              <IS_ZEPPELIN_TEST>true</IS_ZEPPELIN_TEST>
+            </environmentVariables>
             <excludes>
               <exclude>${tests.to.exclude}</exclude>
             </excludes>

python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java

@@ -210,6 +210,22 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
       throw new IOException("Fail to setup JVMGateway\n" + response.getOutput());
     }
 
+    input =
+        getClass().getClassLoader().getResourceAsStream("python/zeppelin_context.py");
+    lines = IOUtils.readLines(input);
+    response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+        .setCode(StringUtils.join(lines, System.lineSeparator())).build());
+    if (response.getStatus() == ExecuteStatus.ERROR) {
+      throw new IOException("Fail to import ZeppelinContext\n" + response.getOutput());
+    }
+
+    response = ipythonClient.block_execute(ExecuteRequest.newBuilder()
+        .setCode("z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)")
+        .build());
+    if (response.getStatus() == ExecuteStatus.ERROR) {
+      throw new IOException("Fail to setup ZeppelinContext\n" + response.getOutput());
+    }
+
     if (additionalPythonInitFile != null) {
       input = getClass().getClassLoader().getResourceAsStream(additionalPythonInitFile);
       lines = IOUtils.readLines(input);

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -65,6 +65,7 @@ import py4j.GatewayServer;
 public class PythonInterpreter extends Interpreter implements ExecuteResultHandler {
   private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreter.class);
   public static final String ZEPPELIN_PYTHON = "python/zeppelin_python.py";
+  public static final String ZEPPELIN_CONTEXT = "python/zeppelin_context.py";
   public static final String ZEPPELIN_PY4JPATH = "interpreter/python/py4j-0.9.2/src";
   public static final String ZEPPELIN_PYTHON_LIBS = "interpreter/lib/python";
   public static final String DEFAULT_ZEPPELIN_PYTHON = "python";
@@ -125,7 +126,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     }
 
     copyFile(out, ZEPPELIN_PYTHON);
-    logger.info("File {} created", scriptPath);
+    // copy zeppelin_context.py as well
+    File zOut = new File(out.getParent() + "/zeppelin_context.py");
+    copyFile(zOut, ZEPPELIN_CONTEXT);
+
+    logger.info("File {} , {} created", scriptPath, zOut.getAbsolutePath());
   }
 
   public String getScriptPath() {
@@ -181,7 +186,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
     cmd.addArgument(getLocalIp(), false);
 
     executor = new DefaultExecutor();
-    outputStream = new InterpreterOutputStream(logger);
+    outputStream = new InterpreterOutputStream(LOG);
     PipedOutputStream ps = new PipedOutputStream();
     in = null;
     try {
@@ -232,30 +237,17 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
         StringUtils.isEmpty(iPythonInterpreter.checkIPythonPrerequisite(getPythonBindPath()))) {
       try {
         iPythonInterpreter.open();
-        if (InterpreterContext.get() != null) {
-          InterpreterContext.get().out.write(("IPython is available, " +
-              "use IPython for PythonInterpreter\n")
-              .getBytes());
-        }
-        LOG.info("Use IPythonInterpreter to replace PythonInterpreter");
+        LOG.info("IPython is available, Use IPythonInterpreter to replace PythonInterpreter");
         return;
       } catch (Exception e) {
         iPythonInterpreter = null;
+        LOG.warn("Fail to open IPythonInterpreter", e);
       }
     }
-    // reset iPythonInterpreter to null
+
+    // reset iPythonInterpreter to null as it is not available
     iPythonInterpreter = null;
-
-    try {
-      if (InterpreterContext.get() != null) {
-        InterpreterContext.get().out.write(("IPython is not available, " +
-            "use the native PythonInterpreter\n")
-            .getBytes());
-      }
-    } catch (IOException e) {
-      LOG.warn("Fail to write InterpreterOutput", e.getMessage());
-    }
-
+    LOG.info("IPython is not available, use the native PythonInterpreter");
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {

python/src/main/resources/grpc/python/zeppelin_python.py

@@ -17,130 +17,8 @@
 
 from py4j.java_gateway import java_import, JavaGateway, GatewayClient
 
-from io import BytesIO
-try:
-  from StringIO import StringIO
-except ImportError:
-  from io import StringIO
-
-class PyZeppelinContext(object):
-  """ A context impl that uses Py4j to communicate to JVM
-  """
-
-  def __init__(self, z):
-    self.z = z
-    self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
-    self.javaList = gateway.jvm.java.util.ArrayList
-    self.max_result = z.getMaxResult()
-
-  def getInterpreterContext(self):
-    return self.z.getInterpreterContext()
-
-  def input(self, name, defaultValue=""):
-    return self.z.input(name, defaultValue)
-
-  def textbox(self, name, defaultValue=""):
-    return self.z.textbox(name, defaultValue)
-
-  def noteTextbox(self, name, defaultValue=""):
-    return self.z.noteTextbox(name, defaultValue)
-
-  def select(self, name, options, defaultValue=""):
-    return self.z.select(name, defaultValue, self.getParamOptions(options))
-
-  def noteSelect(self, name, options, defaultValue=""):
-    return self.z.noteSelect(name, defaultValue, self.getParamOptions(options))
-
-  def checkbox(self, name, options, defaultChecked=[]):
-    return self.z.checkbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
-
-  def noteCheckbox(self, name, options, defaultChecked=[]):
-    return self.z.noteCheckbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
-
-  def getParamOptions(self, options):
-    javaOptions = gateway.new_array(self.paramOption, len(options))
-    i = 0
-    for tuple in options:
-      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
-      i += 1
-    return javaOptions
-
-  def getDefaultChecked(self, defaultChecked):
-    javaDefaultChecked = self.javaList()
-    for check in defaultChecked:
-      javaDefaultChecked.append(check)
-    return javaDefaultChecked
-
-  def show(self, p, **kwargs):
-    if type(p).__name__ == "DataFrame": # does not play well with sub-classes
-      # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
-      # and so a dependency on pandas
-      self.show_dataframe(p, **kwargs)
-    elif hasattr(p, '__call__'):
-      p() #error reporting
-
-  def show_dataframe(self, df, show_index=False, **kwargs):
-    """Pretty prints DF using Table Display System
-    """
-    limit = len(df) > self.max_result
-    header_buf = StringIO("")
-    if show_index:
-      idx_name = str(df.index.name) if df.index.name is not None else ""
-      header_buf.write(idx_name + "\t")
-    header_buf.write(str(df.columns[0]))
-    for col in df.columns[1:]:
-      header_buf.write("\t")
-      header_buf.write(str(col))
-    header_buf.write("\n")
-
-    body_buf = StringIO("")
-    rows = df.head(self.max_result).values if limit else df.values
-    index = df.index.values
-    for idx, row in zip(index, rows):
-      if show_index:
-        body_buf.write("%html <strong>{}</strong>".format(idx))
-        body_buf.write("\t")
-      body_buf.write(str(row[0]))
-      for cell in row[1:]:
-        body_buf.write("\t")
-        body_buf.write(str(cell))
-      body_buf.write("\n")
-    body_buf.seek(0); header_buf.seek(0)
-    #TODO(bzz): fix it, so it shows red notice, as in Spark
-    print("%table " + header_buf.read() + body_buf.read()) # +
-    #      ("\n<font color=red>Results are limited by {}.</font>" \
-    #          .format(self.max_result) if limit else "")
-    #)
-    body_buf.close(); header_buf.close()
-
-  def registerHook(self, event, cmd, replName=None):
-    if replName is None:
-      self.z.registerHook(event, cmd)
-    else:
-      self.z.registerHook(event, cmd, replName)
-
-  def unregisterHook(self, event, replName=None):
-    if replName is None:
-      self.z.unregisterHook(event)
-    else:
-      self.z.unregisterHook(event, replName)
-
-  def registerNoteHook(self, event, cmd, noteId, replName=None):
-    if replName is None:
-      self.z.registerNoteHook(event, cmd, noteId)
-    else:
-      self.z.registerNoteHook(event, cmd, noteId, replName)
-
-  def unregisterNoteHook(self, event, noteId, replName=None):
-    if replName is None:
-      self.z.unregisterNoteHook(event, noteId)
-    else:
-      self.z.unregisterNoteHook(event, noteId, replName)
-
 # start JVM gateway
 client = GatewayClient(address='127.0.0.1', port=${JVM_GATEWAY_PORT})
 gateway = JavaGateway(client)
 java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
 intp = gateway.entry_point
-z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
-

python/src/main/resources/python/zeppelin_context.py

@@ -0,0 +1,224 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, sys
+import warnings
+import base64
+
+from io import BytesIO
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+class PyZeppelinContext(object):
+    """ A context impl that uses Py4j to communicate to JVM
+    """
+
+    def __init__(self, z, gateway):
+        self.z = z
+        self.gateway = gateway
+        self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
+        self.javaList = gateway.jvm.java.util.ArrayList
+        self.max_result = z.getMaxResult()
+        self._displayhook = lambda *args: None
+        self._setup_matplotlib()
+
+    # By implementing special methods it makes operating on it more Pythonic
+    def __setitem__(self, key, item):
+        self.z.put(key, item)
+
+    def __getitem__(self, key):
+        return self.z.get(key)
+
+    def __delitem__(self, key):
+        self.z.remove(key)
+
+    def __contains__(self, item):
+        return self.z.containsKey(item)
+
+    def add(self, key, value):
+        self.__setitem__(key, value)
+
+    def put(self, key, value):
+        self.__setitem__(key, value)
+
+    def get(self, key):
+        return self.__getitem__(key)
+
+    def getInterpreterContext(self):
+        return self.z.getInterpreterContext()
+
+    def input(self, name, defaultValue=""):
+        return self.z.input(name, defaultValue)
+
+    def textbox(self, name, defaultValue=""):
+        return self.z.textbox(name, defaultValue)
+
+    def noteTextbox(self, name, defaultValue=""):
+        return self.z.noteTextbox(name, defaultValue)
+
+    def select(self, name, options, defaultValue=""):
+        return self.z.select(name, defaultValue, self.getParamOptions(options))
+
+    def noteSelect(self, name, options, defaultValue=""):
+        return self.z.noteSelect(name, defaultValue, self.getParamOptions(options))
+
+    def checkbox(self, name, options, defaultChecked=[]):
+        return self.z.checkbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
+
+    def noteCheckbox(self, name, options, defaultChecked=[]):
+        return self.z.noteCheckbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
+
+    def registerHook(self, event, cmd, replName=None):
+        if replName is None:
+            self.z.registerHook(event, cmd)
+        else:
+            self.z.registerHook(event, cmd, replName)
+
+    def unregisterHook(self, event, replName=None):
+        if replName is None:
+            self.z.unregisterHook(event)
+        else:
+            self.z.unregisterHook(event, replName)
+
+    def registerNoteHook(self, event, cmd, noteId, replName=None):
+        if replName is None:
+            self.z.registerNoteHook(event, cmd, noteId)
+        else:
+            self.z.registerNoteHook(event, cmd, noteId, replName)
+
+    def unregisterNoteHook(self, event, noteId, replName=None):
+        if replName is None:
+            self.z.unregisterNoteHook(event, noteId)
+        else:
+            self.z.unregisterNoteHook(event, noteId, replName)
+
+    def getParamOptions(self, options):
+        javaOptions = self.gateway.new_array(self.paramOption, len(options))
+        i = 0
+        for tuple in options:
+            javaOptions[i] = self.paramOption(tuple[0], tuple[1])
+            i += 1
+        return javaOptions
+
+    def getDefaultChecked(self, defaultChecked):
+        javaDefaultChecked = self.javaList()
+        for check in defaultChecked:
+            javaDefaultChecked.append(check)
+        return javaDefaultChecked
+
+    def show(self, p, **kwargs):
+        if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
+            self.show_matplotlib(p, **kwargs)
+        elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
+            # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
+            # and so a dependency on pandas
+            self.show_dataframe(p, **kwargs)
+        else:
+            print(str(p))
+            
+    def show_dataframe(self, df, show_index=False, **kwargs):
+        """Pretty prints DF using Table Display System
+        """
+        exceed_limit = len(df) > self.max_result
+        header_buf = StringIO("")
+        if show_index:
+            idx_name = str(df.index.name) if df.index.name is not None else ""
+            header_buf.write(idx_name + "\t")
+        header_buf.write(str(df.columns[0]))
+        for col in df.columns[1:]:
+            header_buf.write("\t")
+            header_buf.write(str(col))
+        header_buf.write("\n")
+
+        body_buf = StringIO("")
+        rows = df.head(self.max_result).values if exceed_limit else df.values
+        index = df.index.values
+        for idx, row in zip(index, rows):
+            if show_index:
+                body_buf.write("%html <strong>{}</strong>".format(idx))
+                body_buf.write("\t")
+            body_buf.write(str(row[0]))
+            for cell in row[1:]:
+                body_buf.write("\t")
+                body_buf.write(str(cell))
+            body_buf.write("\n")
+        body_buf.seek(0)
+        header_buf.seek(0)
+        print("%table " + header_buf.read() + body_buf.read())
+        body_buf.close(); header_buf.close()
+        if exceed_limit:
+            print("%html <font color=red>Results are limited by {}.</font>".format(self.max_result))
+
+    def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
+                        **kwargs):
+        """Matplotlib show function
+        """
+        if fmt == "png":
+            img = BytesIO()
+            p.savefig(img, format=fmt)
+            img_str = b"data:image/png;base64,"
+            img_str += base64.b64encode(img.getvalue().strip())
+            img_tag = "<img src={img} style='width={width};height:{height}'>"
+            # Decoding is necessary for Python 3 compatibility
+            img_str = img_str.decode("ascii")
+            img_str = img_tag.format(img=img_str, width=width, height=height)
+        elif fmt == "svg":
+            img = StringIO()
+            p.savefig(img, format=fmt)
+            img_str = img.getvalue()
+        else:
+            raise ValueError("fmt must be 'png' or 'svg'")
+
+        html = "%html <div style='width:{width};height:{height}'>{img}<div>"
+        print(html.format(width=width, height=height, img=img_str))
+        img.close()
+
+    def configure_mpl(self, **kwargs):
+        import mpl_config
+        mpl_config.configure(**kwargs)
+
+    def _setup_matplotlib(self):
+        # If we don't have matplotlib installed don't bother continuing
+        try:
+            import matplotlib
+        except ImportError:
+            return
+
+        # Make sure custom backends are available in the PYTHONPATH
+        rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
+        mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
+        if mpl_path not in sys.path:
+            sys.path.append(mpl_path)
+
+        # Finally check if backend exists, and if so configure as appropriate
+        try:
+            matplotlib.use('module://backend_zinline')
+            import backend_zinline
+
+            # Everything looks good so make config assuming that we are using
+            # an inline backend
+            self._displayhook = backend_zinline.displayhook
+            self.configure_mpl(width=600, height=400, dpi=72, fontsize=10,
+                               interactive=True, format='png', context=self.z)
+        except ImportError:
+            # Fall back to Agg if no custom backend installed
+            matplotlib.use('Agg')
+            warnings.warn("Unable to load inline matplotlib backend, "
+                          "falling back to Agg")

python/src/main/resources/python/zeppelin_python.py

@@ -47,182 +47,6 @@ class Logger(object):
   def flush(self):
     pass
 
-
-class PyZeppelinContext(object):
-  """ A context impl that uses Py4j to communicate to JVM
-  """
-
-  def __init__(self, z):
-    self.z = z
-    self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
-    self.javaList = gateway.jvm.java.util.ArrayList
-    self.max_result = 1000
-    self._displayhook = lambda *args: None
-    self._setup_matplotlib()
-
-  def getInterpreterContext(self):
-    return self.z.getInterpreterContext()
-
-  def input(self, name, defaultValue=""):
-    return self.z.input(name, defaultValue)
-
-  def textbox(self, name, defaultValue=""):
-    return self.z.textbox(name, defaultValue)
-
-  def noteTextbox(self, name, defaultValue=""):
-    return self.z.noteTextbox(name, defaultValue)
-
-  def select(self, name, options, defaultValue=""):
-    return self.z.select(name, defaultValue, self.getParamOptions(options))
-
-  def noteSelect(self, name, options, defaultValue=""):
-    return self.z.noteSelect(name, defaultValue, self.getParamOptions(options))
-
-  def checkbox(self, name, options, defaultChecked=[]):
-    return self.z.checkbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
-
-  def noteCheckbox(self, name, options, defaultChecked=[]):
-    return self.z.noteCheckbox(name, self.getDefaultChecked(defaultChecked), self.getParamOptions(options))
-
-  def registerHook(self, event, cmd, replName=None):
-    if replName is None:
-      self.z.registerHook(event, cmd)
-    else:
-      self.z.registerHook(event, cmd, replName)
-
-  def unregisterHook(self, event, replName=None):
-    if replName is None:
-      self.z.unregisterHook(event)
-    else:
-      self.z.unregisterHook(event, replName)
-
-  def registerNoteHook(self, event, cmd, noteId, replName=None):
-    if replName is None:
-      self.z.registerNoteHook(event, cmd, noteId)
-    else:
-      self.z.registerNoteHook(event, cmd, noteId, replName)
-
-  def unregisterNoteHook(self, event, noteId, replName=None):
-    if replName is None:
-      self.z.unregisterNoteHook(event, noteId)
-    else:
-      self.z.unregisterNoteHook(event, noteId, replName)
-
-  def getParamOptions(self, options):
-    javaOptions = gateway.new_array(self.paramOption, len(options))
-    i = 0
-    for tuple in options:
-      javaOptions[i] = self.paramOption(tuple[0], tuple[1])
-      i += 1
-    return javaOptions
-
-  def getDefaultChecked(self, defaultChecked):
-    javaDefaultChecked = self.javaList()
-    for check in defaultChecked:
-      javaDefaultChecked.append(check)
-    return javaDefaultChecked
-
-  def show(self, p, **kwargs):
-    if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
-      self.show_matplotlib(p, **kwargs)
-    elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
-      # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
-      # and so a dependency on pandas
-      self.show_dataframe(p, **kwargs)
-    elif hasattr(p, '__call__'):
-      p() #error reporting
-
-  def show_dataframe(self, df, show_index=False, **kwargs):
-    """Pretty prints DF using Table Display System
-    """
-    limit = len(df) > self.max_result
-    header_buf = StringIO("")
-    if show_index:
-      idx_name = str(df.index.name) if df.index.name is not None else ""
-      header_buf.write(idx_name + "\t")
-    header_buf.write(str(df.columns[0]))
-    for col in df.columns[1:]:
-      header_buf.write("\t")
-      header_buf.write(str(col))
-    header_buf.write("\n")
-
-    body_buf = StringIO("")
-    rows = df.head(self.max_result).values if limit else df.values
-    index = df.index.values
-    for idx, row in zip(index, rows):
-      if show_index:
-        body_buf.write("%html <strong>{}</strong>".format(idx))
-        body_buf.write("\t")
-      body_buf.write(str(row[0]))
-      for cell in row[1:]:
-        body_buf.write("\t")
-        body_buf.write(str(cell))
-      body_buf.write("\n")
-    body_buf.seek(0); header_buf.seek(0)
-    #TODO(bzz): fix it, so it shows red notice, as in Spark
-    print("%table " + header_buf.read() + body_buf.read()) # +
-    #      ("\n<font color=red>Results are limited by {}.</font>" \
-    #          .format(self.max_result) if limit else "")
-    #)
-    body_buf.close(); header_buf.close()
-
-  def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
-                      **kwargs):
-    """Matplotlib show function
-    """
-    if fmt == "png":
-      img = BytesIO()
-      p.savefig(img, format=fmt)
-      img_str = b"data:image/png;base64,"
-      img_str += base64.b64encode(img.getvalue().strip())
-      img_tag = "<img src={img} style='width={width};height:{height}'>"
-      # Decoding is necessary for Python 3 compability
-      img_str = img_str.decode("ascii")
-      img_str = img_tag.format(img=img_str, width=width, height=height)
-    elif fmt == "svg":
-      img = StringIO()
-      p.savefig(img, format=fmt)
-      img_str = img.getvalue()
-    else:
-      raise ValueError("fmt must be 'png' or 'svg'")
-
-    html = "%html <div style='width:{width};height:{height}'>{img}<div>"
-    print(html.format(width=width, height=height, img=img_str))
-    img.close()
-
-  def configure_mpl(self, **kwargs):
-    import mpl_config
-    mpl_config.configure(**kwargs)
-
-  def _setup_matplotlib(self):
-    # If we don't have matplotlib installed don't bother continuing
-    try:
-      import matplotlib
-    except ImportError:
-      return
-    # Make sure custom backends are available in the PYTHONPATH
-    rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
-    mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
-    if mpl_path not in sys.path:
-      sys.path.append(mpl_path)
-
-    # Finally check if backend exists, and if so configure as appropriate
-    try:
-      matplotlib.use('module://backend_zinline')
-      import backend_zinline
-
-      # Everything looks good so make config assuming that we are using
-      # an inline backend
-      self._displayhook = backend_zinline.displayhook
-      self.configure_mpl(width=600, height=400, dpi=72,
-                         fontsize=10, interactive=True, format='png')
-    except ImportError:
-      # Fall back to Agg if no custom backend installed
-      matplotlib.use('Agg')
-      warnings.warn("Unable to load inline matplotlib backend, "
-                    "falling back to Agg")
-
-
 def handler_stop_signals(sig, frame):
   sys.exit("Got signal : " + str(sig))
 
@@ -236,14 +60,16 @@ if len(sys.argv) >= 3:
 _zcUserQueryNameSpace = {}
 client = GatewayClient(address=host, port=int(sys.argv[1]))
 
-#gateway = JavaGateway(client, auto_convert = True)
 gateway = JavaGateway(client)
 
 intp = gateway.entry_point
 intp.onPythonScriptInitialized(os.getpid())
 
 java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
-z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
+
+from zeppelin_context import PyZeppelinContext
+
+z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)
 __zeppelin__._setup_matplotlib()
 
 _zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__

python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java

@@ -71,14 +71,16 @@ public class IPythonInterpreterTest {
 
   @Test
   public void testIPython() throws IOException, InterruptedException, InterpreterException {
-    startInterpreter(new Properties());
+    Properties properties = new Properties();
+    properties.setProperty("zeppelin.python.maxResult", "3");
+    startInterpreter(properties);
     testInterpreter(interpreter);
   }
 
   @Test
   public void testGrpcFrameSize() throws InterpreterException, IOException {
     Properties properties = new Properties();
-    properties.setProperty("zeppelin.ipython.grpc.message_size", "4");
+    properties.setProperty("zeppelin.ipython.grpc.message_size", "200");
     startInterpreter(properties);
 
     // to make this test can run under both python2 and python3
@@ -86,11 +88,11 @@ public class IPythonInterpreterTest {
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
 
     InterpreterContext context = getInterpreterContext();
-    result = interpreter.interpret("print(11111111111111111111111111111)", context);
+    result = interpreter.interpret("print('1'*300)", context);
     assertEquals(InterpreterResult.Code.ERROR, result.code());
     List<InterpreterResultMessage> interpreterResultMessages = context.out.toInterpreterResultMessage();
     assertEquals(1, interpreterResultMessages.size());
-    assertTrue(interpreterResultMessages.get(0).getData().contains("Frame size 32 exceeds maximum: 4"));
+    assertTrue(interpreterResultMessages.get(0).getData().contains("Frame size 304 exceeds maximum: 200"));
 
     // next call continue work
     result = interpreter.interpret("print(1)", context);
@@ -99,14 +101,14 @@ public class IPythonInterpreterTest {
     close();
 
     // increase framesize to make it work
-    properties.setProperty("zeppelin.ipython.grpc.message_size", "40");
+    properties.setProperty("zeppelin.ipython.grpc.message_size", "500");
     startInterpreter(properties);
     // to make this test can run under both python2 and python3
     result = interpreter.interpret("from __future__ import print_function", getInterpreterContext());
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
 
     context = getInterpreterContext();
-    result = interpreter.interpret("print(11111111111111111111111111111)", context);
+    result = interpreter.interpret("print('1'*300)", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
   }
 
@@ -454,9 +456,29 @@ public class IPythonInterpreterTest {
     result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
     interpreterResultMessages = context.out.toInterpreterResultMessage();
+    assertEquals(1, interpreterResultMessages.size());
     assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType());
     assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData());
 
+    context = getInterpreterContext();
+    result = interpreter.interpret("import pandas as pd\ndf = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.toInterpreterResultMessage();
+    assertEquals(2, interpreterResultMessages.size());
+    assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType());
+    assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData());
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
+    assertEquals("<font color=red>Results are limited by 3.</font>\n", interpreterResultMessages.get(1).getData());
+
+    // z.show(matplotlib)
+    context = getInterpreterContext();
+    result = interpreter.interpret("import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", context);
+    assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+    interpreterResultMessages = context.out.toInterpreterResultMessage();
+    assertEquals(2, interpreterResultMessages.size());
+    assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
+    assertEquals(InterpreterResult.Type.IMG, interpreterResultMessages.get(1).getType());
+
     // clear output
     context = getInterpreterContext();
     result = interpreter.interpret("import time\nprint(\"Hello\")\ntime.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context);

python/src/test/resources/log4j.properties

@@ -29,3 +29,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
 log4j.rootLogger=INFO, stdout
 log4j.logger.org.apache.zeppelin.python.IPythonInterpreter=DEBUG
 log4j.logger.org.apache.zeppelin.python.IPythonClient=DEBUG
+log4j.logger.org.apache.zeppelin.python=DEBUG

shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java

@@ -85,7 +85,9 @@ public class ShellInterpreter extends KerberosInterpreter {
 
 
   @Override
-  public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
+  public InterpreterResult interpret(String originalCmd, InterpreterContext contextInterpreter) {
+    String cmd = Boolean.parseBoolean(getProperty("zeppelin.shell.interpolation")) ?
+            interpolate(originalCmd, contextInterpreter.getResourcePool()) : originalCmd;
     LOGGER.debug("Run shell command '" + cmd + "'");
     OutputStream outStream = new ByteArrayOutputStream();
     

shell/src/main/resources/interpreter-setting.json

@@ -38,6 +38,13 @@
         "defaultValue": "",
         "description": "Kerberos principal",
         "type": "string"
+      },
+      "zeppelin.shell.interpolation": {
+        "envName": null,
+        "propertyName": "zeppelin.shell.interpolation",
+        "defaultValue": false,
+        "description": "Enable ZeppelinContext variable interpolation into paragraph text",
+        "type": "checkbox"
       }
     },
     "editor": {

spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -114,6 +114,16 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
       throw new InterpreterException(e);
     }
 
+    try {
+      FileOutputStream outStream = new FileOutputStream(out.getParent() + "/zeppelin_context.py");
+      IOUtils.copy(
+          classLoader.getResourceAsStream("python/zeppelin_context.py"),
+          outStream);
+      outStream.close();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
+
     LOGGER.info("File {} created", scriptPath);
   }
 
@@ -126,30 +136,17 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
             iPySparkInterpreter.checkIPythonPrerequisite(getPythonExec(getProperties())))) {
       try {
         iPySparkInterpreter.open();
-        if (InterpreterContext.get() != null) {
-          // don't print it when it is in testing, just for easy output check in test.
-          InterpreterContext.get().out.write(("IPython is available, " +
-              "use IPython for PySparkInterpreter\n")
-              .getBytes());
-        }
-        LOGGER.info("Use IPySparkInterpreter to replace PySparkInterpreter");
+        LOGGER.info("IPython is available, Use IPySparkInterpreter to replace PySparkInterpreter");
         return;
       } catch (Exception e) {
+        iPySparkInterpreter = null;
         LOGGER.warn("Fail to open IPySparkInterpreter", e);
       }
     }
-    iPySparkInterpreter = null;
-    if (getProperty("zeppelin.pyspark.useIPython", "true").equals("true")) {
-      // don't print it when it is in testing, just for easy output check in test.
-      try {
-        InterpreterContext.get().out.write(("IPython is not available, " +
-            "use the native PySparkInterpreter\n")
-            .getBytes());
-      } catch (IOException e) {
-        LOGGER.warn("Fail to write InterpreterOutput", e);
-      }
-    }
 
+    // reset iPySparkInterpreter to null as it is not available
+    iPySparkInterpreter = null;
+    LOGGER.info("IPython is not available, use the native PySparkInterpreter\n");
     // Add matplotlib display hook
     InterpreterGroup intpGroup = getInterpreterGroup();
     if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {

spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java

@@ -115,7 +115,9 @@ public class SparkSqlInterpreter extends Interpreter {
       // to    def sql(sqlText: String): DataFrame (1.3 and later).
       // Therefore need to use reflection to keep binary compatibility for all spark versions.
       Method sqlMethod = sqlc.getClass().getMethod("sql", String.class);
-      rdd = sqlMethod.invoke(sqlc, st);
+      String effectiveString = Boolean.parseBoolean(getProperty("zeppelin.spark.sql.interpolation")) ?
+              interpolate(st, context.getResourcePool()) : st;
+      rdd = sqlMethod.invoke(sqlc, effectiveString);
     } catch (InvocationTargetException ite) {
       if (Boolean.parseBoolean(getProperty("zeppelin.spark.sql.stacktrace"))) {
         throw new InterpreterException(ite);

spark/interpreter/src/main/resources/interpreter-setting.json

@@ -109,6 +109,13 @@
         "description": "Show full exception stacktrace for SQL queries if set to true.",
         "type": "checkbox"
       },
+      "zeppelin.spark.sql.interpolation": {
+        "envName": null,
+        "propertyName": "zeppelin.spark.sql.interpolation",
+        "defaultValue": false,
+        "description": "Enable ZeppelinContext variable interpolation into paragraph text",
+        "type": "checkbox"
+      },
       "zeppelin.spark.maxResult": {
         "envName": "ZEPPELIN_SPARK_MAXRESULT",
         "propertyName": "zeppelin.spark.maxResult",

spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py

@@ -54,8 +54,8 @@ else:
 
 class IPySparkZeppelinContext(PyZeppelinContext):
 
-    def __init__(self, z):
-        super(IPySparkZeppelinContext, self).__init__(z)
+    def __init__(self, z, gateway):
+        super(IPySparkZeppelinContext, self).__init__(z, gateway)
 
     def show(self, obj):
         from pyspark.sql import DataFrame
@@ -64,4 +64,4 @@ class IPySparkZeppelinContext(PyZeppelinContext):
         else:
             super(IPySparkZeppelinContext, self).show(obj)
 
-z = __zeppelin__ = IPySparkZeppelinContext(intp.getZeppelinContext())
+z = __zeppelin__ = IPySparkZeppelinContext(intp.getZeppelinContext(), gateway)

spark/interpreter/src/main/resources/python/zeppelin_pyspark.py

@@ -41,155 +41,6 @@ class Logger(object):
     pass
 
 
-class PyZeppelinContext(dict):
-  def __init__(self, zc):
-    self.z = zc
-    self._displayhook = lambda *args: None
-
-  def show(self, obj):
-    from pyspark.sql import DataFrame
-    if isinstance(obj, DataFrame):
-      print(self.z.showData(obj._jdf))
-    else:
-      print(str(obj))
-
-  # By implementing special methods it makes operating on it more Pythonic
-  def __setitem__(self, key, item):
-    self.z.put(key, item)
-
-  def __getitem__(self, key):
-    return self.z.get(key)
-
-  def __delitem__(self, key):
-    self.z.remove(key)
-
-  def __contains__(self, item):
-    return self.z.containsKey(item)
-
-  def add(self, key, value):
-    self.__setitem__(key, value)
-
-  def put(self, key, value):
-    self.__setitem__(key, value)
-
-  def get(self, key):
-    return self.__getitem__(key)
-
-  def getInterpreterContext(self):
-    return self.z.getInterpreterContext()
-
-  def input(self, name, defaultValue=""):
-    return self.z.input(name, defaultValue)
-
-  def textbox(self, name, defaultValue=""):
-    return self.z.textbox(name, defaultValue)
-
-  def noteTextbox(self, name, defaultValue=""):
-    return self.z.noteTextbox(name, defaultValue)
-
-  def select(self, name, options, defaultValue=""):
-    # auto_convert to ArrayList doesn't match the method signature on JVM side
-    return self.z.select(name, defaultValue, self.getParamOptions(options))
-
-  def noteSelect(self, name, options, defaultValue=""):
-    return self.z.noteSelect(name, defaultValue, self.getParamOptions(options))
-
-  def checkbox(self, name, options, defaultChecked=None):
-    optionsIterable = self.getParamOptions(options)
-    defaultCheckedIterables = self.getDefaultChecked(defaultChecked)
-    checkedItems = gateway.jvm.scala.collection.JavaConversions.seqAsJavaList(self.z.checkbox(name, defaultCheckedIterables, optionsIterable))
-    result = []
-    for checkedItem in checkedItems:
-      result.append(checkedItem)
-    return result;
-
-  def noteCheckbox(self, name, options, defaultChecked=None):
-    optionsIterable = self.getParamOptions(options)
-    defaultCheckedIterables = self.getDefaultChecked(defaultChecked)
-    checkedItems = gateway.jvm.scala.collection.JavaConversions.seqAsJavaList(self.z.noteCheckbox(name, defaultCheckedIterables, optionsIterable))
-    result = []
-    for checkedItem in checkedItems:
-      result.append(checkedItem)
-    return result;
-
-  def getParamOptions(self, options):
-    tuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options))
-    return gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(tuples)
-
-  def getDefaultChecked(self, defaultChecked):
-    if defaultChecked is None:
-      defaultChecked = []
-    return gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(defaultChecked)
-
-  def registerHook(self, event, cmd, replName=None):
-    if replName is None:
-      self.z.registerHook(event, cmd)
-    else:
-      self.z.registerHook(event, cmd, replName)
-
-  def unregisterHook(self, event, replName=None):
-    if replName is None:
-      self.z.unregisterHook(event)
-    else:
-      self.z.unregisterHook(event, replName)
-
-  def registerNoteHook(self, event, cmd, noteId, replName=None):
-    if replName is None:
-      self.z.registerNoteHook(event, cmd, noteId)
-    else:
-      self.z.registerNoteHook(event, cmd, noteId, replName)
-
-  def unregisterNoteHook(self, event, noteId, replName=None):
-    if replName is None:
-      self.z.unregisterNoteHook(event, noteId)
-    else:
-      self.z.unregisterNoteHook(event, noteId, replName)
-
-  def getHook(self, event, replName=None):
-    if replName is None:
-      return self.z.getHook(event)
-    return self.z.getHook(event, replName)
-
-  def _setup_matplotlib(self):
-    # If we don't have matplotlib installed don't bother continuing
-    try:
-      import matplotlib
-    except ImportError:
-      return
-    
-    # Make sure custom backends are available in the PYTHONPATH
-    rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
-    mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
-    if mpl_path not in sys.path:
-      sys.path.append(mpl_path)
-    
-    # Finally check if backend exists, and if so configure as appropriate
-    try:
-      matplotlib.use('module://backend_zinline')
-      import backend_zinline
-      
-      # Everything looks good so make config assuming that we are using
-      # an inline backend
-      self._displayhook = backend_zinline.displayhook
-      self.configure_mpl(width=600, height=400, dpi=72, fontsize=10,
-                         interactive=True, format='png', context=self.z)
-    except ImportError:
-      # Fall back to Agg if no custom backend installed
-      matplotlib.use('Agg')
-      warnings.warn("Unable to load inline matplotlib backend, "
-                    "falling back to Agg")
-
-  def configure_mpl(self, **kwargs):
-    import mpl_config
-    mpl_config.configure(**kwargs)
-
-  def __tupleToScalaTuple2(self, tuple):
-    if (len(tuple) == 2):
-      return gateway.jvm.scala.Tuple2(tuple[0], tuple[1])
-    else:
-      raise IndexError("options must be a list of tuple of 2")
-
-
 class SparkVersion(object):
   SPARK_1_4_0 = 10400
   SPARK_1_3_0 = 10300
@@ -322,7 +173,24 @@ completion = __zeppelin_completion__ = PySparkCompletion(intp)
 _zcUserQueryNameSpace["completion"] = completion
 _zcUserQueryNameSpace["__zeppelin_completion__"] = __zeppelin_completion__
 
-z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
+
+from zeppelin_context import PyZeppelinContext
+
+#TODO(zjffdu) merge it with IPySparkZeppelinContext
+class PySparkZeppelinContext(PyZeppelinContext):
+
+  def __init__(self, z, gateway):
+    super(PySparkZeppelinContext, self).__init__(z, gateway)
+
+  def show(self, obj):
+    from pyspark.sql import DataFrame
+    if isinstance(obj, DataFrame):
+      print(self.z.showData(obj._jdf))
+    else:
+      super(PySparkZeppelinContext, self).show(obj)
+
+z = __zeppelin__ = PySparkZeppelinContext(intp.getZeppelinContext(), gateway)
+
 __zeppelin__._setup_matplotlib()
 _zcUserQueryNameSpace["z"] = z
 _zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__

spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java

@@ -64,7 +64,7 @@ public class IPySparkInterpreterTest {
     p.setProperty("spark.submit.deployMode", "client");
     p.setProperty("spark.app.name", "Zeppelin Test");
     p.setProperty("zeppelin.spark.useHiveContext", "true");
-    p.setProperty("zeppelin.spark.maxResult", "1000");
+    p.setProperty("zeppelin.spark.maxResult", "3");
     p.setProperty("zeppelin.spark.importImplicit", "true");
     p.setProperty("zeppelin.pyspark.python", "python");
     p.setProperty("zeppelin.dep.localrepo", Files.createTempDir().getAbsolutePath());

zeppelin-distribution/src/assemble/distribution.xml

@@ -91,6 +91,9 @@
     <fileSet>
       <directory>../notebook</directory>
     </fileSet>
+    <fileSet>
+      <directory>../plugins</directory>
+    </fileSet>
     <fileSet>
       <outputDirectory>/lib/interpreter</outputDirectory>
       <directory>../zeppelin-interpreter/target/lib</directory>

zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -347,10 +347,18 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
   }
 
+  public String getPluginsDir() {
+    return getRelativeDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR));
+  }
+
   public String getRecoveryDir() {
     return getRelativeDir(ConfVars.ZEPPELIN_RECOVERY_DIR);
   }
 
+  public String getNotebookStorageClass() {
+    return getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE);
+  }
+
   public String getRecoveryStorageClass() {
     return getString(ConfVars.ZEPPELIN_RECOVERY_STORAGE_CLASS);
   }
@@ -711,6 +719,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_RECOVERY_DIR("zeppelin.recovery.dir", "recovery"),
     ZEPPELIN_RECOVERY_STORAGE_CLASS("zeppelin.recovery.storage.class",
         "org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage"),
+    ZEPPELIN_PLUGINS_DIR("zeppelin.plugins.dir", "plugins"),
 
     // use specified notebook (id) as homescreen
     ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -23,6 +23,8 @@ import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.zeppelin.annotation.Experimental;
 import org.apache.zeppelin.annotation.ZeppelinApi;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.resource.Resource;
+import org.apache.zeppelin.resource.ResourcePool;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
 import org.slf4j.Logger;
@@ -37,6 +39,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 /**
  * Interface for interpreters.
  * If you want to implement new Zeppelin interpreter, extend this class
@@ -78,6 +83,35 @@ public abstract class Interpreter {
     return null;
   }
 
+  protected String interpolate(String cmd, ResourcePool resourcePool) {
+    Pattern zVariablePattern = Pattern.compile("([^{}]*)([{]+[^{}]*[}]+)(.*)", Pattern.DOTALL);
+    StringBuilder sb = new StringBuilder();
+    Matcher m;
+    String st = cmd;
+    while ((m = zVariablePattern.matcher(st)).matches()) {
+      sb.append(m.group(1));
+      String varPat = m.group(2);
+      if (varPat.matches("[{][^{}]+[}]")) {
+        // substitute {variable} only if 'variable' has a value ...
+        Resource resource = resourcePool.get(varPat.substring(1, varPat.length() - 1));
+        Object variableValue = resource == null ? null : resource.get();
+        if (variableValue != null)
+          sb.append(variableValue);
+        else
+          return cmd;
+      } else if (varPat.matches("[{]{2}[^{}]+[}]{2}")) {
+        // escape {{text}} ...
+        sb.append("{").append(varPat.substring(2, varPat.length() - 2)).append("}");
+      } else {
+        // mismatched {{ }} or more than 2 braces ...
+        return cmd;
+      }
+      st = m.group(3);
+    }
+    sb.append(st);
+    return sb.toString();
+  }
+
   /**
    * Run code and return result, in synchronous way.
    *

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/ZeppCtxtVariableTest.java

@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.resource.ResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.Before;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+public class ZeppCtxtVariableTest {
+
+  public static class TestInterpreter extends Interpreter {
+
+    TestInterpreter(Properties property) {
+      super(property);
+    }
+
+    @Override
+    public void open() {
+    }
+
+    @Override
+    public void close() {
+    }
+
+    @Override
+    public InterpreterResult interpret(String st, InterpreterContext context) {
+      return null;
+    }
+
+    @Override
+    public void cancel(InterpreterContext context) {
+    }
+
+    @Override
+    public FormType getFormType() {
+      return null;
+    }
+
+    @Override
+    public int getProgress(InterpreterContext context) {
+      return 0;
+    }
+  }
+
+  private Interpreter interpreter;
+  private ResourcePool resourcePool;
+
+  @Before
+  public void setUp() throws Exception {
+
+    resourcePool = new LocalResourcePool("ZeppelinContextVariableInterpolationTest");
+
+    InterpreterContext.set(new InterpreterContext("InterpolationTestNoteId",
+            "InterpolationTestParagraphTitle",
+            null,
+            "InterpolationTestParagraphTitle",
+            "InterpolationTestParagraphText",
+            new AuthenticationInfo("InterpolationTestUser", null, "testTicket"),
+            null,
+            null,
+            null,
+            null,
+            resourcePool,
+            null,
+            null));
+
+    interpreter = new TestInterpreter(new Properties());
+
+    resourcePool.put("PI", "3.1415");
+
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    InterpreterContext.remove();
+  }
+
+  @Test
+  public void stringWithoutPatterns() {
+    String result = interpreter.interpolate("The value of PI is not exactly 3.14", resourcePool);
+    assertTrue("String without patterns", "The value of PI is not exactly 3.14".equals(result));
+  }
+
+  @Test
+  public void substitutionInTheMiddle() {
+    String result = interpreter.interpolate("The value of {{PI}} is {PI} now", resourcePool);
+    assertTrue("Substitution in the middle", "The value of {PI} is 3.1415 now".equals(result));
+  }
+
+  @Test
+  public void substitutionAtTheEnds() {
+    String result = interpreter.interpolate("{{PI}} is now {PI}", resourcePool);
+    assertTrue("Substitution at the ends", "{PI} is now 3.1415".equals(result));
+  }
+
+  @Test
+  public void multiLineSubstitutionSuccessful1() {
+    String result = interpreter.interpolate("{{PI}}\n{PI}\n{{PI}}\n{PI}", resourcePool);
+    assertTrue("multiLineSubstitutionSuccessful1", "{PI}\n3.1415\n{PI}\n3.1415".equals(result));
+  }
+
+
+  @Test
+  public void multiLineSubstitutionSuccessful2() {
+    String result = interpreter.interpolate("prefix {PI} {{PI\n}} suffix", resourcePool);
+    assertTrue("multiLineSubstitutionSuccessful2", "prefix 3.1415 {PI\n} suffix".equals(result));
+  }
+
+
+  @Test
+  public void multiLineSubstitutionSuccessful3() {
+    String result = interpreter.interpolate("prefix {{\nPI}} {PI} suffix", resourcePool);
+    assertTrue("multiLineSubstitutionSuccessful3", "prefix {\nPI} 3.1415 suffix".equals(result));
+  }
+
+
+  @Test
+  public void multiLineSubstitutionFailure2() {
+    String result = interpreter.interpolate("prefix {PI\n} suffix", resourcePool);
+    assertTrue("multiLineSubstitutionFailure2", "prefix {PI\n} suffix".equals(result));
+  }
+
+
+  @Test
+  public void multiLineSubstitutionFailure3() {
+    String result = interpreter.interpolate("prefix {\nPI} suffix", resourcePool);
+    assertTrue("multiLineSubstitutionFailure3", "prefix {\nPI} suffix".equals(result));
+  }
+
+  @Test
+  public void noUndefinedVariableError() {
+    String result = interpreter.interpolate("This {pi} will pass silently", resourcePool);
+    assertTrue("No partial substitution", "This {pi} will pass silently".equals(result));
+  }
+
+  @Test
+  public void noPartialSubstitution() {
+    String result = interpreter.interpolate("A {PI} and a {PIE} are different", resourcePool);
+    assertTrue("No partial substitution", "A {PI} and a {PIE} are different".equals(result));
+  }
+
+  @Test
+  public void substitutionAndEscapeMixed() {
+    String result = interpreter.interpolate("A {PI} is not a {{PIE}}", resourcePool);
+    assertTrue("Substitution and escape mixed", "A 3.1415 is not a {PIE}".equals(result));
+  }
+
+  @Test
+  public void unbalancedBracesOne() {
+    String result = interpreter.interpolate("A {PI} and a {{PIE} remain unchanged", resourcePool);
+    assertTrue("Unbalanced braces - one", "A {PI} and a {{PIE} remain unchanged".equals(result));
+  }
+
+  @Test
+  public void unbalancedBracesTwo() {
+    String result = interpreter.interpolate("A {PI} and a {PIE}} remain unchanged", resourcePool);
+    assertTrue("Unbalanced braces - one", "A {PI} and a {PIE}} remain unchanged".equals(result));
+  }
+
+  @Test
+  public void tooManyBraces() {
+    String result = interpreter.interpolate("This {{{PI}}} remain unchanged", resourcePool);
+    assertTrue("Too many braces", "This {{{PI}}} remain unchanged".equals(result));
+  }
+
+  @Test
+  public void randomBracesOne() {
+    String result = interpreter.interpolate("A {{ starts an escaped sequence", resourcePool);
+    assertTrue("Random braces - one", "A {{ starts an escaped sequence".equals(result));
+  }
+
+  @Test
+  public void randomBracesTwo() {
+    String result = interpreter.interpolate("A }} ends an escaped sequence", resourcePool);
+    assertTrue("Random braces - two", "A }} ends an escaped sequence".equals(result));
+  }
+
+  @Test
+  public void randomBracesThree() {
+    String result = interpreter.interpolate("Paired { begin an escaped sequence", resourcePool);
+    assertTrue("Random braces - three", "Paired { begin an escaped sequence".equals(result));
+  }
+
+  @Test
+  public void randomBracesFour() {
+    String result = interpreter.interpolate("Paired } end an escaped sequence", resourcePool);
+    assertTrue("Random braces - four", "Paired } end an escaped sequence".equals(result));
+  }
+
+}

zeppelin-plugins/notebookrepo/azure/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-azure</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin AzureNotebookRepo</name>
+    <description>NotebookRepo implementation based on Azure</description>
+
+    <properties>
+        <adl.sdk.version>2.1.4</adl.sdk.version>
+        <azure.storage.version>4.0.0</azure.storage.version>
+        <plugin.name>NotebookRepo/AzureNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-data-lake-store-sdk</artifactId>
+            <version>${adl.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-storage</artifactId>
+            <version>${azure.storage.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-lang3</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java

@@ -48,32 +48,39 @@ import org.slf4j.LoggerFactory;
  * Azure storage backend for notebooks
  */
 public class AzureNotebookRepo implements NotebookRepo {
-  private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class);
+  private static final Logger LOG = LoggerFactory.getLogger(AzureNotebookRepo.class);
 
-  private final ZeppelinConfiguration conf;
-  private final String user;
-  private final String shareName;
-  private final CloudFileDirectory rootDir;
+  private ZeppelinConfiguration conf;
+  private String user;
+  private String shareName;
+  private CloudFileDirectory rootDir;
 
-  public AzureNotebookRepo(ZeppelinConfiguration conf)
-      throws URISyntaxException, InvalidKeyException, StorageException {
+  public AzureNotebookRepo() {
+
+  }
+
+  public void init(ZeppelinConfiguration conf) throws IOException {
     this.conf = conf;
     user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
     shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);
 
-    CloudStorageAccount account = CloudStorageAccount.parse(
-        conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
-    CloudFileClient client = account.createCloudFileClient();
-    CloudFileShare share = client.getShareReference(shareName);
-    share.createIfNotExists();
+    try {
+      CloudStorageAccount account = CloudStorageAccount.parse(
+          conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
+      CloudFileClient client = account.createCloudFileClient();
+      CloudFileShare share = client.getShareReference(shareName);
+      share.createIfNotExists();
 
-    CloudFileDirectory userDir = StringUtils.isBlank(user) ?
-        share.getRootDirectoryReference() :
-        share.getRootDirectoryReference().getDirectoryReference(user);
-    userDir.createIfNotExists();
+      CloudFileDirectory userDir = StringUtils.isBlank(user) ?
+          share.getRootDirectoryReference() :
+          share.getRootDirectoryReference().getDirectoryReference(user);
+      userDir.createIfNotExists();
 
-    rootDir = userDir.getDirectoryReference("notebook");
-    rootDir.createIfNotExists();
+      rootDir = userDir.getDirectoryReference("notebook");
+      rootDir.createIfNotExists();
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
   }
 
   @Override

zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.AzureNotebookRepo

zeppelin-plugins/notebookrepo/filesystem/pom.xml

@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-filesystem</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin FileSystemNotebookRepo</name>
+    <description>NotebookRepo implementation based on Hadoop FileSystem</description>
+
+    <properties>
+        <adl.sdk.version>2.1.4</adl.sdk.version>
+        <plugin.name>NotebookRepo/FileSystemNotebookRepo</plugin.name>
+    </properties>
+
+    <profiles>
+
+        <profile>
+            <id>hadoop2-azure</id>
+            <properties>
+                <hadoop.version>2.7.3</hadoop.version>
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-azure</artifactId>
+                    <version>${hadoop.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.google.guava</groupId>
+                            <artifactId>guava</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.apache.commons</groupId>
+                            <artifactId>commons-lang3</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.jcraf</groupId>
+                            <artifactId>jsch</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.apache.commons</groupId>
+                            <artifactId>commons-compress</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+                <dependency>
+                    <groupId>com.microsoft.azure</groupId>
+                    <artifactId>azure-data-lake-store-sdk</artifactId>
+                    <version>${adl.sdk.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop2-aws</id>
+            <properties>
+                <hadoop.version>2.7.3</hadoop.version>
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-aws</artifactId>
+                    <version>${hadoop.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-annotations</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-databind</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>joda-time</groupId>
+                            <artifactId>joda-time</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop3-azure</id>
+            <properties>
+                <hadoop.version>3.0.0</hadoop.version>
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-azure</artifactId>
+                    <version>${hadoop.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.google.guava</groupId>
+                            <artifactId>guava</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.jcraft</groupId>
+                            <artifactId>jsch</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.apache.commons</groupId>
+                            <artifactId>commons-compress</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.codehaus.jackson</groupId>
+                            <artifactId>jackson-mapper-asl</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.nimbusds</groupId>
+                            <artifactId>nimbus-jose-jwt</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.apache.zookeeper</groupId>
+                            <artifactId>zookeeper</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.eclipse.jetty</groupId>
+                            <artifactId>jetty-server</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.eclipse.jetty</groupId>
+                            <artifactId>jetty-servlet</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.codehaus.jackson</groupId>
+                            <artifactId>jackson-core-asl</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-databind</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>org.eclipse.jetty</groupId>
+                            <artifactId>jetty-util</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.sun.jersey</groupId>
+                            <artifactId>jersey-core</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-azure-datalake</artifactId>
+                    <version>${hadoop.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <profile>
+            <id>hadoop3-aws</id>
+            <properties>
+                <hadoop.version>3.0.0</hadoop.version>
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-aws</artifactId>
+                    <version>${hadoop.version}</version>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-annotations</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-core</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>com.fasterxml.jackson.core</groupId>
+                            <artifactId>jackson-databind</artifactId>
+                        </exclusion>
+                        <exclusion>
+                            <groupId>joda-time</groupId>
+                            <artifactId>joda-time</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
+</project>

zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java

@@ -41,7 +41,11 @@ public class FileSystemNotebookRepo implements NotebookRepo {
   private FileSystemStorage fs;
   private Path notebookDir;
 
-  public FileSystemNotebookRepo(ZeppelinConfiguration zConf) throws IOException {
+  public FileSystemNotebookRepo() {
+
+  }
+
+  public void init(ZeppelinConfiguration zConf) throws IOException {
     this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir());
     LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() +
         " for Zeppelin Notebook.");

zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo

zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java

@@ -37,7 +37,8 @@ public class FileSystemNotebookRepoTest {
     System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir);
     hadoopConf = new Configuration();
     fs = FileSystem.get(hadoopConf);
-    hdfsNotebookRepo = new FileSystemNotebookRepo(zConf);
+    hdfsNotebookRepo = new FileSystemNotebookRepo();
+    hdfsNotebookRepo.init(zConf);
   }
 
   @After

zeppelin-plugins/notebookrepo/gcs/pom.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-gcs</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin GCSNotebookRepo</name>
+    <description>NotebookRepo implementation based on Google Cloud Storage</description>
+
+    <properties>
+        <gcs.storage.version>1.14.0</gcs.storage.version>
+        <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version>
+        <google.truth.version>0.27</google.truth.version>
+        <plugin.name>NotebookRepo/GCSNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>${gcs.storage.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.api</groupId>
+                    <artifactId>api-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.http-client</groupId>
+                    <artifactId>google-http-client-jackson2</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.http-client</groupId>
+                    <artifactId>google-http-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.codehaus.jackson</groupId>
+                    <artifactId>jackson-core-asl</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-nio</artifactId>
+            <version>${google.testing.nio.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.truth</groupId>
+            <artifactId>truth</artifactId>
+            <version>${google.truth.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.api</groupId>
+            <artifactId>api-common</artifactId>
+            <version>1.2.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client-jackson2</artifactId>
+            <version>1.23.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java

@@ -24,6 +24,7 @@ import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.Storage.BlobListOption;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -66,15 +67,20 @@ public class GCSNotebookRepo implements NotebookRepo {
   private Pattern noteNamePattern;
   private Storage storage;
 
-  public GCSNotebookRepo(ZeppelinConfiguration conf) throws IOException {
-    this(conf, StorageOptions.getDefaultInstance().getService());
+  public GCSNotebookRepo() {
   }
 
-  // For tests to use an in-memory storage implementation
-  GCSNotebookRepo(ZeppelinConfiguration conf, Storage storage) throws IOException {
-    this.encoding =  conf.getString(ConfVars.ZEPPELIN_ENCODING);
+  @VisibleForTesting
+  public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException {
+    init(zConf);
+    this.storage = storage;
+  }
 
-    String gcsStorageDir = conf.getGCSStorageDir();
+  @Override
+  public void init(ZeppelinConfiguration zConf) throws IOException {
+    this.encoding =  zConf.getString(ConfVars.ZEPPELIN_ENCODING);
+
+    String gcsStorageDir = zConf.getGCSStorageDir();
     if (gcsStorageDir.isEmpty()) {
       throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'");
     }
@@ -107,7 +113,7 @@ public class GCSNotebookRepo implements NotebookRepo {
       this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
     }
 
-    this.storage = storage;
+    this.storage = StorageOptions.getDefaultInstance().getService();
   }
 
   private BlobId makeBlobId(String noteId) {

zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GCSNotebookRepo

zeppelin-plugins/notebookrepo/git/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-git</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin GitNotebookRepo</name>
+    <description>NotebookRepo implementation based on Git</description>
+
+    <properties>
+        <eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version>
+        <google.truth.version>0.27</google.truth.version>
+        <plugin.name>NotebookRepo/GitNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.zeppelin</groupId>
+            <artifactId>notebookrepo-vfs</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit</artifactId>
+            <version>${eclipse.jgit.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.truth</groupId>
+            <artifactId>truth</artifactId>
+            <version>${google.truth.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java

@@ -17,11 +17,9 @@
 
 package org.apache.zeppelin.notebook.repo;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.user.AuthenticationInfo;
@@ -39,8 +37,10 @@ import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
 
 /**
  * NotebookRepo that hosts all the notebook FS in a single Git repo
@@ -48,18 +48,33 @@ import com.google.common.collect.Lists;
  * This impl intended to be simple and straightforward:
  *   - does not handle branches
  *   - only basic local git file repo, no remote Github push\pull. GitHub integration is
- *   implemented in @see {@link org.apache.zeppelin.notebook.repo.GitHubNotebookRepo}
+ *   implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo}
  *
  *   TODO(bzz): add default .gitignore
  */
-class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl {
+public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl {
   private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class);
 
   private String localPath;
   private Git git;
 
+  public GitNotebookRepo() {
+    super();
+  }
+
+  @VisibleForTesting
   public GitNotebookRepo(ZeppelinConfiguration conf) throws IOException {
-    super(conf);
+    this();
+    init(conf);
+  }
+
+  @Override
+  public void init(ZeppelinConfiguration conf) throws IOException {
+    //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause
+    //AbstractMethodError
+    this.conf = conf;
+    setNotebookDirectory(conf.getNotebookDir());
+
     localPath = getRootDir().getName().getPath();
     LOG.info("Opening a git repo at '{}'", localPath);
     Repository localRepo = new FileRepository(Joiner.on(File.separator).join(localPath, ".git"));

zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GitNotebookRepo

zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java

@@ -72,14 +72,14 @@ public class GitNotebookRepoTest {
     String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2);
     FileUtils.copyDirectory(
             new File(
-              GitHubNotebookRepoTest.class.getResource(
+                GitNotebookRepoTest.class.getResource(
                 Joiner.on(File.separator).join("", TEST_NOTE_ID)
               ).getFile()
             ),
             new File(testNoteDir));
     FileUtils.copyDirectory(
             new File(
-              GitHubNotebookRepoTest.class.getResource(
+                GitNotebookRepoTest.class.getResource(
                 Joiner.on(File.separator).join("", TEST_NOTE_ID2)
               ).getFile()
             ),
@@ -193,7 +193,7 @@ public class GitNotebookRepoTest {
     List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
     assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isNotEmpty();
     int initialCount = notebookHistoryBefore.size();
-    
+
     // add changes to note
     Note note = notebookRepo.get(TEST_NOTE_ID, null);
     note.setInterpreterFactory(mock(InterpreterFactory.class));
@@ -202,16 +202,16 @@ public class GitNotebookRepoTest {
     config.put("enabled", true);
     p.setConfig(config);
     p.setText("%md checkpoint test text");
-    
+
     // save and checkpoint note
     notebookRepo.save(note, null);
     notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null);
-    
+
     // see if commit is added
     List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
     assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1);
   }
-  
+
   private boolean containsNote(List<NoteInfo> notes, String noteId) {
     for (NoteInfo note: notes) {
       if (note.getId().equals(noteId)) {
@@ -328,19 +328,19 @@ public class GitNotebookRepoTest {
     assertThat(notebookRepo.list(null)).isNotEmpty();
     assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
     assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
-    
+
     // get current note
     Note note = notebookRepo.get(TEST_NOTE_ID, null);
     note.setInterpreterFactory(mock(InterpreterFactory.class));
     int paragraphCount_1 = note.getParagraphs().size();
     LOG.info("initial paragraph count: {}", paragraphCount_1);
-    
+
     // checkpoint revision1
     Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null);
     //TODO(khalid): change to EMPTY after rebase
     assertThat(revision1).isNotNull();
     assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1);
-    
+
     // add one more paragraph and save
     Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
     Map<String, Object> config = p1.getConfig();
@@ -351,33 +351,33 @@ public class GitNotebookRepoTest {
     int paragraphCount_2 = note.getParagraphs().size();
     assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1);
     LOG.info("paragraph count after modification: {}", paragraphCount_2);
-    
+
     // checkpoint revision2
     Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null);
     //TODO(khalid): change to EMPTY after rebase
     assertThat(revision2).isNotNull();
     assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2);
-    
+
     // set note to revision1
     Note returnedNote = notebookRepo.setNoteRevision(note.getId(), revision1.id, null);
     assertThat(returnedNote).isNotNull();
     assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_1);
-    
+
     // check note from repo
     Note updatedNote = notebookRepo.get(note.getId(), null);
     assertThat(updatedNote).isNotNull();
     assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_1);
-    
+
     // set back to revision2
     returnedNote = notebookRepo.setNoteRevision(note.getId(), revision2.id, null);
     assertThat(returnedNote).isNotNull();
     assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_2);
-    
+
     // check note from repo
     updatedNote = notebookRepo.get(note.getId(), null);
     assertThat(updatedNote).isNotNull();
     assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_2);
-    
+
     // try failure case - set to invalid revision
     returnedNote = notebookRepo.setNoteRevision(note.getId(), "nonexistent_id", null);
     assertThat(returnedNote).isNull();

zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java

@@ -17,12 +17,6 @@
 
 package org.apache.zeppelin.notebook.repo;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
 import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock;
@@ -32,6 +26,13 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+//TODO(zjffdu) move it to zeppelin-zengine
 public class NotebookRepoSyncInitializationTest {
   private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class);
   private String validFirstStorageClass = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo";
@@ -42,15 +43,16 @@ public class NotebookRepoSyncInitializationTest {
   private String invalidTwoStorageConf = validFirstStorageClass + "," + invalidStorageClass;
   private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass;
   private String emptyStorageConf = "";
-  
+
   @Before
   public void setUp(){
+    System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath());
     //setup routine
   }
-  
+
   @After
   public void tearDown() {
-    //tear-down routine 
+    //tear-down routine
   }
 
   @Test
@@ -79,7 +81,7 @@ public class NotebookRepoSyncInitializationTest {
     File secNotebookDir = new File(secNotePath);
     mainNotebookDir.mkdirs();
     secNotebookDir.mkdirs();
-    
+
     // set confs
     System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
     System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
@@ -92,7 +94,7 @@ public class NotebookRepoSyncInitializationTest {
     assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
     assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock);
   }
-  
+
   @Test
   public void invalidInitTwoStorageTest() throws IOException {
     // set confs
@@ -105,7 +107,7 @@ public class NotebookRepoSyncInitializationTest {
     assertEquals(notebookRepoSync.getRepoCount(), 1);
     assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
   }
-  
+
   @Test
   public void initUnsupportedNumberStoragesTest() throws IOException {
     // initialize folders for each storage, currently for 2 only
@@ -119,7 +121,7 @@ public class NotebookRepoSyncInitializationTest {
     File secNotebookDir = new File(secNotePath);
     mainNotebookDir.mkdirs();
     secNotebookDir.mkdirs();
-    
+
     // set confs
     System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
     System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
@@ -127,7 +129,7 @@ public class NotebookRepoSyncInitializationTest {
     ZeppelinConfiguration conf = ZeppelinConfiguration.create();
     // create repo
     NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
-    // check that first two storages initialized instead of three 
+    // check that first two storages initialized instead of three
     assertEquals(notebookRepoSync.getRepoCount(), 2);
     assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
     assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock);
@@ -144,7 +146,7 @@ public class NotebookRepoSyncInitializationTest {
     assertEquals(notebookRepoSync.getRepoCount(), 1);
     assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepoWithVersionControl);
   }
-  
+
   @Test
   public void initOneDummyStorageTest() throws IOException {
  // set confs

zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java

@@ -17,18 +17,6 @@
 
 package org.apache.zeppelin.notebook.repo;
 
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
@@ -39,7 +27,12 @@ import org.apache.zeppelin.interpreter.InterpreterFactory;
 import org.apache.zeppelin.interpreter.InterpreterResultMessage;
 import org.apache.zeppelin.interpreter.InterpreterSettingManager;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
-import org.apache.zeppelin.notebook.*;
+import org.apache.zeppelin.notebook.JobListenerFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Notebook;
+import org.apache.zeppelin.notebook.NotebookAuthorization;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.notebook.ParagraphJobListener;
 import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.scheduler.Job.Status;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
@@ -54,7 +47,20 @@ import org.quartz.SchedulerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+
+//TODO(zjffdu) move it to zeppelin-zengine
 public class NotebookRepoSyncTest implements JobListenerFactory {
 
   private File mainZepDir;
@@ -91,6 +97,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
     System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock");
     System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false");
     System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf");
+    System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath());
 
     LOG.info("main Note dir : " + mainNotePath);
     LOG.info("secondary note dir : " + secNotePath);

zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java

@@ -16,27 +16,21 @@
  */
 package org.apache.zeppelin.notebook.repo.mock;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.commons.vfs2.VFS;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
 import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
 
+import java.io.IOException;
+
 public class VFSNotebookRepoMock extends VFSNotebookRepo {
 
-  private static ZeppelinConfiguration modifyNotebookDir(ZeppelinConfiguration conf) {
-    String secNotebookDir = conf.getNotebookDir() + "_secondary";
+  public VFSNotebookRepoMock() {
+    String secNotebookDir = ZeppelinConfiguration.create().getNotebookDir() + "_secondary";
     System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir);
-    ZeppelinConfiguration secConf = ZeppelinConfiguration.create();
-    return secConf;
   }
 
-  public VFSNotebookRepoMock(ZeppelinConfiguration conf) throws IOException {
-    super(modifyNotebookDir(conf));
+  public void init(ZeppelinConfiguration conf) throws IOException {
+    super.init(conf);
   }
 
 }

zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=
+#%5p [%t] (%F:%L) - %m%n
+#%-4r [%t] %-5p %c %x - %m%n
+#
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+log4j.logger.org.apache.zeppelin.notebook.repo=DEBUG
+ 
+#mute some noisy guys
+log4j.logger.org.apache.hadoop.mapred=WARN
+log4j.logger.org.apache.hadoop.hive.ql=WARN
+log4j.logger.org.apache.hadoop.hive.metastore=WARN
+log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
+
+log4j.logger.org.quartz=WARN
+log4j.logger.DataNucleus=WARN
+log4j.logger.DataNucleus.MetaData=ERROR
+log4j.logger.DataNucleus.Datastore=ERROR
+
+# Log all JDBC parameters
+log4j.logger.org.hibernate.type=ALL
+
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
+log4j.logger.org.apache.zeppelin.plugin=DEBUG
+

zeppelin-plugins/notebookrepo/github/pom.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-github</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin GitHubNotebookRepo</name>
+    <description>NotebookRepo implementation based on GitHub</description>
+
+    <properties>
+        <plugin.name>NotebookRepo/GitHubNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.zeppelin</groupId>
+            <artifactId>notebookrepo-git</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GitHubNotebookRepo

zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java

@@ -49,7 +49,7 @@ import static org.mockito.Mockito.mock;
  * 2. The second repository is considered as the local notebook repository
  */
 public class GitHubNotebookRepoTest {
-  private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class);
+  private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepoTest.class);
 
   private static final String TEST_NOTE_ID = "2A94M5J1Z";
 

zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json

@@ -0,0 +1,341 @@
+{
+  "paragraphs": [
+    {
+      "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "editorHide": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1423836981412_-1007008116",
+      "id": "20150213-231621_168813393",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n"
+      },
+      "dateCreated": "Feb 13, 2015 11:16:21 PM",
+      "dateStarted": "Apr 1, 2015 9:11:09 PM",
+      "dateFinished": "Apr 1, 2015 9:11:10 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "title": "Load data into table",
+      "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n    IOUtils.toString(\n        new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n        Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n    s \u003d\u003e Bank(s(0).toInt, \n            s(1).replaceAll(\"\\\"\", \"\"),\n            s(2).replaceAll(\"\\\"\", \"\"),\n            s(3).replaceAll(\"\\\"\", \"\"),\n            s(5).replaceAll(\"\\\"\", \"\").toInt\n        )\n).toDF()\nbank.registerTempTable(\"bank\")",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "title": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1423500779206_-1502780787",
+      "id": "20150210-015259_1403135953",
+      "result": {
+        "code": "SUCCESS",
+        "type": "TEXT",
+        "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
+      },
+      "dateCreated": "Feb 10, 2015 1:52:59 AM",
+      "dateStarted": "Jul 3, 2015 1:43:40 PM",
+      "dateFinished": "Jul 3, 2015 1:43:45 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
+      "config": {
+        "colWidth": 4.0,
+        "graph": {
+          "mode": "multiBarChart",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          }
+        }
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1423500782552_-1439281894",
+      "id": "20150210-015302_1492795503",
+      "result": {
+        "code": "SUCCESS",
+        "type": "TABLE",
+        "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
+      },
+      "dateCreated": "Feb 10, 2015 1:53:02 AM",
+      "dateStarted": "Jul 3, 2015 1:43:17 PM",
+      "dateFinished": "Jul 3, 2015 1:43:23 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
+      "config": {
+        "colWidth": 4.0,
+        "graph": {
+          "mode": "multiBarChart",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          }
+        }
+      },
+      "settings": {
+        "params": {
+          "maxAge": "35"
+        },
+        "forms": {
+          "maxAge": {
+            "name": "maxAge",
+            "defaultValue": "30",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1423720444030_-1424110477",
+      "id": "20150212-145404_867439529",
+      "result": {
+        "code": "SUCCESS",
+        "type": "TABLE",
+        "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
+      },
+      "dateCreated": "Feb 12, 2015 2:54:04 PM",
+      "dateStarted": "Jul 3, 2015 1:43:28 PM",
+      "dateFinished": "Jul 3, 2015 1:43:29 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
+      "config": {
+        "colWidth": 4.0,
+        "graph": {
+          "mode": "multiBarChart",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "age",
+              "index": 0.0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "value",
+              "index": 1.0,
+              "aggr": "sum"
+            }
+          }
+        }
+      },
+      "settings": {
+        "params": {
+          "marital": "single"
+        },
+        "forms": {
+          "marital": {
+            "name": "marital",
+            "defaultValue": "single",
+            "options": [
+              {
+                "value": "single"
+              },
+              {
+                "value": "divorced"
+              },
+              {
+                "value": "married"
+              }
+            ],
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1423836262027_-210588283",
+      "id": "20150213-230422_1600658137",
+      "result": {
+        "code": "SUCCESS",
+        "type": "TABLE",
+        "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
+      },
+      "dateCreated": "Feb 13, 2015 11:04:22 PM",
+      "dateStarted": "Jul 3, 2015 1:43:33 PM",
+      "dateFinished": "Jul 3, 2015 1:43:34 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "editorHide": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1423836268492_216498320",
+      "id": "20150213-230428_1231780373",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+      },
+      "dateCreated": "Feb 13, 2015 11:04:28 PM",
+      "dateStarted": "Apr 1, 2015 9:12:18 PM",
+      "dateFinished": "Apr 1, 2015 9:12:18 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n  This dataset is public available for research. The details are described in [Moro et al., 2011]. \n  Please include this citation if you plan to use this database:\n\n  [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n  In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n  Available at: [pdf] http://hdl.handle.net/1822/14838\n                [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "editorHide": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1427420818407_872443482",
+      "id": "20150326-214658_12335843",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n  This dataset is public available for research. The details are described in [Moro et al., 2011]. \n  Please include this citation if you plan to use this database:\n\n  [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n  In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n  Available at: [pdf] http://hdl.handle.net/1822/14838\n                [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+      },
+      "dateCreated": "Mar 26, 2015 9:46:58 PM",
+      "dateStarted": "Jul 3, 2015 1:44:56 PM",
+      "dateFinished": "Jul 3, 2015 1:44:56 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "config": {},
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1435955447812_-158639899",
+      "id": "20150703-133047_853701097",
+      "dateCreated": "Jul 3, 2015 1:30:47 PM",
+      "status": "READY",
+      "progressUpdateIntervalMs": 500
+    }
+  ],
+  "name": "Zeppelin Tutorial",
+  "id": "2A94M5J1Z",
+  "angularObjects": {},
+  "config": {
+    "looknfeel": "default"
+  },
+  "info": {}
+}

zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json

@@ -0,0 +1,87 @@
+{
+  "paragraphs": [
+    {
+      "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "editorHide": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1423836268492_216498320",
+      "id": "20150213-230428_1231780373",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+      },
+      "dateCreated": "Feb 13, 2015 11:04:28 PM",
+      "dateStarted": "Apr 1, 2015 9:12:18 PM",
+      "dateFinished": "Apr 1, 2015 9:12:18 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n  This dataset is public available for research. The details are described in [Moro et al., 2011]. \n  Please include this citation if you plan to use this database:\n\n  [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n  In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n  Available at: [pdf] http://hdl.handle.net/1822/14838\n                [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+      "config": {
+        "colWidth": 12.0,
+        "graph": {
+          "mode": "table",
+          "height": 300.0,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "editorHide": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1427420818407_872443482",
+      "id": "20150326-214658_12335843",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n  This dataset is public available for research. The details are described in [Moro et al., 2011]. \n  Please include this citation if you plan to use this database:\n\n  [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n  In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n  Available at: [pdf] http://hdl.handle.net/1822/14838\n                [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+      },
+      "dateCreated": "Mar 26, 2015 9:46:58 PM",
+      "dateStarted": "Jul 3, 2015 1:44:56 PM",
+      "dateFinished": "Jul 3, 2015 1:44:56 PM",
+      "status": "FINISHED",
+      "progressUpdateIntervalMs": 500
+    },
+    {
+      "config": {},
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1435955447812_-158639899",
+      "id": "20150703-133047_853701097",
+      "dateCreated": "Jul 3, 2015 1:30:47 PM",
+      "status": "READY",
+      "progressUpdateIntervalMs": 500
+    }
+  ],
+  "name": "Sample note - excerpt from Zeppelin Tutorial",
+  "id": "2A94M5J2Z",
+  "angularObjects": {},
+  "config": {
+    "looknfeel": "default"
+  },
+  "info": {}
+}

zeppelin-plugins/notebookrepo/mongodb/pom.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-mongodb</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin MongoNotebookRepo</name>
+    <description>NotebookRepo implementation based on Mongodb</description>
+
+    <properties>
+        <plugin.name>NotebookRepo/MongoNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.zeppelin</groupId>
+            <artifactId>notebookrepo-vfs</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongo-java-driver</artifactId>
+            <version>3.4.1</version>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java

@@ -36,12 +36,16 @@ import org.slf4j.LoggerFactory;
 public class MongoNotebookRepo implements NotebookRepo {
   private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class);
 
-  private final ZeppelinConfiguration conf;
-  private final MongoClient mongo;
-  private final MongoDatabase db;
-  private final MongoCollection<Document> coll;
+  private ZeppelinConfiguration conf;
+  private MongoClient mongo;
+  private MongoDatabase db;
+  private MongoCollection<Document> coll;
 
-  public MongoNotebookRepo(ZeppelinConfiguration conf) throws IOException {
+  public MongoNotebookRepo() {
+
+  }
+
+  public void init(ZeppelinConfiguration conf) throws IOException {
     this.conf = conf;
 
     mongo = new MongoClient(new MongoClientURI(conf.getMongoUri()));
@@ -61,7 +65,8 @@ public class MongoNotebookRepo implements NotebookRepo {
    */
   private void insertFileSystemNotes() throws IOException {
     LinkedList<Document> docs = new LinkedList<>(); // docs to be imported
-    NotebookRepo vfsRepo = new VFSNotebookRepo(this.conf);
+    NotebookRepo vfsRepo = new VFSNotebookRepo();
+    vfsRepo.init(conf);
     List<NoteInfo> infos =  vfsRepo.list(null);
     // collect notes to be imported
     for (NoteInfo info : infos) {

zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.MongoNotebookRepo

zeppelin-plugins/notebookrepo/s3/pom.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-s3</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin S3NotebookRepo</name>
+    <description>NotebookRepo implementation based on S3</description>
+
+    <properties>
+        <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version>
+        <plugin.name>NotebookRepo/S3NotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${aws.sdk.s3.version}</version>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java

@@ -82,13 +82,17 @@ public class S3NotebookRepo implements NotebookRepo {
   //  3. Credential profiles file at the default location (~/.aws/credentials)
   //       shared by all AWS SDKs and the AWS CLI
   //  4. Instance profile credentials delivered through the Amazon EC2 metadata service
-  private final AmazonS3 s3client;
-  private final String bucketName;
-  private final String user;
-  private final boolean useServerSideEncryption;
-  private final ZeppelinConfiguration conf;
+  private AmazonS3 s3client;
+  private String bucketName;
+  private String user;
+  private boolean useServerSideEncryption;
+  private ZeppelinConfiguration conf;
 
-  public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
+  public S3NotebookRepo() {
+
+  }
+
+  public void init(ZeppelinConfiguration conf) throws IOException {
     this.conf = conf;
     bucketName = conf.getS3BucketName();
     user = conf.getS3User();

zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.S3NotebookRepo

zeppelin-plugins/notebookrepo/vfs/pom.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-vfs</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin VFSNotebookRepo</name>
+    <description>NotebookRepo implementation based on VFS</description>
+
+    <properties>
+        <commons.vfs2.version>2.2</commons.vfs2.version>
+        <plugin.name>NotebookRepo/VFSNotebookRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-vfs2</artifactId>
+            <version>${commons.vfs2.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.codehaus.plexus</groupId>
+                    <artifactId>plexus-utils</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+</project>

zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java

@@ -24,6 +24,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -53,15 +54,21 @@ public class VFSNotebookRepo implements NotebookRepo {
 
   private FileSystemManager fsManager;
   private URI filesystemRoot;
-  private ZeppelinConfiguration conf;
+  protected ZeppelinConfiguration conf;
 
-  public VFSNotebookRepo(ZeppelinConfiguration conf) throws IOException {
+  public VFSNotebookRepo() {
+
+  }
+
+  @Override
+  public void init(ZeppelinConfiguration conf) throws IOException {
     this.conf = conf;
     setNotebookDirectory(conf.getNotebookDir());
   }
 
-  private void setNotebookDirectory(String notebookDirPath) throws IOException {
+  protected void setNotebookDirectory(String notebookDirPath) throws IOException {
     try {
+      LOG.info("Using notebookDir: " + notebookDirPath);
       if (conf.isWindowsPath(notebookDirPath)) {
         filesystemRoot = new File(notebookDirPath).toURI();
       } else {
@@ -241,8 +248,7 @@ public class VFSNotebookRepo implements NotebookRepo {
   @Override
   public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
     NotebookRepoSettingsInfo repoSetting = NotebookRepoSettingsInfo.newInstance();
-    List<NotebookRepoSettingsInfo> settings = Lists.newArrayList();
-
+    List<NotebookRepoSettingsInfo> settings = new ArrayList<>();
     repoSetting.name = "Notebook Path";
     repoSetting.type = NotebookRepoSettingsInfo.Type.INPUT;
     repoSetting.value = Collections.emptyList();

zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.VFSNotebookRepo

zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestVFSNotebookRepo {
+
+  private ZeppelinConfiguration zConf;
+  private VFSNotebookRepo notebookRepo;
+  private String notebookDir = "/tmp/zeppelin/vfs_notebookrepo/";
+
+  @Before
+  public void setUp() throws IOException {
+    notebookRepo = new VFSNotebookRepo();
+    FileUtils.forceMkdir(new File(notebookDir));
+    System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir);
+    zConf = new ZeppelinConfiguration();
+    notebookRepo.init(zConf);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    FileUtils.deleteDirectory(new File(notebookDir));
+  }
+
+  @Test
+  public void testBasics() throws IOException {
+    assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+    Note note1 = new Note();
+    Paragraph p1 = note1.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS);
+    p1.setText("%md hello world");
+    p1.setTitle("my title");
+    notebookRepo.save(note1, AuthenticationInfo.ANONYMOUS);
+
+    assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+    Note note2 = notebookRepo.get(note1.getId(), AuthenticationInfo.ANONYMOUS);
+    assertEquals(note1.getParagraphCount(), note2.getParagraphCount());
+
+    Paragraph p2 = note2.getParagraph(p1.getId());
+    assertEquals(p1.getText(), p2.getText());
+    assertEquals(p1.getTitle(), p2.getTitle());
+
+    notebookRepo.remove(note1.getId(), AuthenticationInfo.ANONYMOUS);
+    assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+  }
+
+  @Test
+  public void testInvalidJson() throws IOException {
+    assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+    // invalid note will be ignored
+    createNewNote("invalid_content", "id_1");
+    assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+    // only valid note will be fetched
+    createNewNote("{}", "id_2");
+    assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+  }
+
+  @Test
+  public void testUpdateSettings() throws IOException {
+    List<NotebookRepoSettingsInfo> repoSettings = notebookRepo.getSettings(AuthenticationInfo.ANONYMOUS);
+    assertEquals(1, repoSettings.size());
+    NotebookRepoSettingsInfo settingInfo = repoSettings.get(0);
+    assertEquals("Notebook Path", settingInfo.name);
+    assertEquals(notebookDir, settingInfo.selected);
+
+    createNewNote("{}", "id_2");
+    assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+    String newNotebookDir = "/tmp/zeppelin/vfs_notebookrepo2";
+    FileUtils.forceMkdir(new File(newNotebookDir));
+    Map<String, String> newSettings = ImmutableMap.of("Notebook Path", newNotebookDir);
+    notebookRepo.updateSettings(newSettings, AuthenticationInfo.ANONYMOUS);
+    assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+  }
+
+  private void createNewNote(String content, String noteId) throws IOException {
+    FileUtils.writeStringToFile(new File(notebookDir + "/" + noteId, "note.json"), content);
+  }
+}

zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zengine-plugins-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../../../zeppelin-plugins</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>notebookrepo-zeppelin-hub</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugin ZeppelinHubRepo</name>
+    <description>NotebookRepo implementation based on Zeppelin Hub</description>
+
+    <properties>
+        <jetty.version>9.2.15.v20160210</jetty.version>
+        <google.truth.version>0.27</google.truth.version>
+        <plugin.name>NotebookRepo/ZeppelinHubRepo</plugin.name>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-server</artifactId>
+            <version>${jetty.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-servlet</artifactId>
+            <version>${jetty.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.eclipse.jetty.websocket</groupId>
+            <artifactId>websocket-server</artifactId>
+            <version>${jetty.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.truth</groupId>
+            <artifactId>truth</artifactId>
+            <version>${google.truth.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java

@@ -53,18 +53,27 @@ public class ZeppelinHubRepo implements NotebookRepoWithVersionControl {
   private static final String DEFAULT_SERVER = "https://www.zeppelinhub.com";
   static final String ZEPPELIN_CONF_PROP_NAME_SERVER = "zeppelinhub.api.address";
   static final String ZEPPELIN_CONF_PROP_NAME_TOKEN = "zeppelinhub.api.token";
-  public static final String TOKEN_HEADER = "X-Zeppelin-Token";
+
   private static final Gson GSON = new Gson();
   private static final Note EMPTY_NOTE = new Note();
-  private final Client websocketClient;
-  private final UserTokenContainer tokenManager;
+  private Client websocketClient;
+  private UserTokenContainer tokenManager;
 
   private String token;
   private ZeppelinhubRestApiHandler restApiClient;
   
-  private final ZeppelinConfiguration conf;
-  
+  private ZeppelinConfiguration conf;
+
+  public ZeppelinHubRepo() {
+
+  }
+
   public ZeppelinHubRepo(ZeppelinConfiguration conf) {
+    this();
+    init(conf);
+  }
+
+  public void init(ZeppelinConfiguration conf) {
     this.conf = conf;
     String zeppelinHubUrl = getZeppelinHubUrl(conf);
     LOG.info("Initializing ZeppelinHub integration module");

zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo

zeppelin-plugins/notebookrepo/zeppelin-hub/src/main/test/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java

@@ -35,7 +35,8 @@ public class ZeppelinHubRepoTest {
     System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_TOKEN, token);
 
     ZeppelinConfiguration conf = new ZeppelinConfiguration();
-    repo = new ZeppelinHubRepo(conf);
+    repo = new ZeppelinHubRepo();
+    repo.init(conf);
     repo.setZeppelinhubRestApiHandler(getMockedZeppelinHandler());
   }
 

zeppelin-plugins/pom.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zeppelin</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>..</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>zengine-plugins-parent</artifactId>
+    <packaging>pom</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Plugins Parent</name>
+    <description>Zeppelin Plugins Parent</description>
+
+    <modules>
+        <module>notebookrepo/s3</module>
+        <module>notebookrepo/vfs</module>
+        <module>notebookrepo/git</module>
+        <module>notebookrepo/github</module>
+        <module>notebookrepo/azure</module>
+        <module>notebookrepo/gcs</module>
+        <module>notebookrepo/mongodb</module>
+        <module>notebookrepo/zeppelin-hub</module>
+        <module>notebookrepo/filesystem</module>
+    </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>zeppelin-zengine</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <!-- Test libraries -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-plugin-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/../../../../plugins/${plugin.name}</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                            <includeScope>runtime</includeScope>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>copy-plugin-artifact</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/../../../../plugins/${plugin.name}</outputDirectory>
+                            <overWriteReleases>false</overWriteReleases>
+                            <overWriteSnapshots>false</overWriteSnapshots>
+                            <overWriteIfNewer>true</overWriteIfNewer>
+                            <artifactItems>
+                                <artifactItem>
+                                    <groupId>${project.groupId}</groupId>
+                                    <artifactId>${project.artifactId}</artifactId>
+                                    <version>${project.version}</version>
+                                    <type>${project.packaging}</type>
+                                </artifactItem>
+                            </artifactItems>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+        </plugins>
+    </build>
+</project>

zeppelin-server/pom.xml

@@ -418,42 +418,6 @@
         </executions>
       </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>start-zeppelin</id>
-            <phase>pre-integration-test</phase>
-            <configuration>
-              <target unless="skipTests">
-                <exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}" spawn="true">
-                  <arg value="start" />
-                </exec>
-              </target>
-            </configuration>
-            <goals>
-              <goal>run</goal>
-            </goals>
-          </execution>
-
-          <execution>
-            <id>stop-zeppelin</id>
-            <phase>post-integration-test</phase>
-            <configuration>
-              <target unless="skipTests">
-                <exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}" spawn="false">
-                  <arg value="stop" />
-                </exec>
-              </target>
-            </configuration>
-            <goals>
-              <goal>run</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>

zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java

@@ -43,7 +43,7 @@ import java.util.Set;
  */
 public class PamRealm extends AuthorizingRealm {
 
-  private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubRealm.class);
+  private static final Logger LOG = LoggerFactory.getLogger(PamRealm.class);
 
   private String service;
 

zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java

@@ -88,23 +88,23 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
     LOG.debug("{} successfully login via ZeppelinHub", user.login);
     return new SimpleAuthenticationInfo(user.login, token.getPassword(), name);
   }
-  
+
   @Override
   protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
     // TODO(xxx): future work will be done here.
     return null;
   }
-  
+
   protected void onInit() {
     super.onInit();
   }
-  
+
   /**
    * Setter of ZeppelinHub URL, this will be called by Shiro based on zeppelinhubUrl property
    * in shiro.ini file.</p>
-   * It will also perform a check of ZeppelinHub url {@link #isZeppelinHubUrlValid}, 
+   * It will also perform a check of ZeppelinHub url {@link #isZeppelinHubUrlValid},
    * if the url is not valid, the default zeppelinhub url will be used.
-   * 
+   *
    * @param url
    */
   public void setZeppelinhubUrl(String url) {
@@ -118,9 +118,9 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
   }
 
   /**
-   * Send to ZeppelinHub a login request based on the request body which is a JSON that contains 2 
+   * Send to ZeppelinHub a login request based on the request body which is a JSON that contains 2
    * fields "login" and "password".
-   * 
+   *
    * @param requestBody JSON string of ZeppelinHub payload.
    * @return Account object with login, name (if set in ZeppelinHub), and mail.
    * @throws AuthenticationException if fail to login.
@@ -141,12 +141,12 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
       responseBody = put.getResponseBodyAsString();
       userSession = put.getResponseHeader(USER_SESSION_HEADER).getValue();
       put.releaseConnection();
-      
+
     } catch (IOException e) {
       LOG.error("Cannot login user", e);
       throw new AuthenticationException(e.getMessage());
     }
-    
+
     User account = null;
     try {
       account = User.fromJson(responseBody);
@@ -156,7 +156,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
     }
 
     onLoginSuccess(account.login, userSession);
-    
+
     return account;
   }
 
@@ -182,7 +182,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
    * Perform a Simple URL check by using <code>URI(url).toURL()</code>.
    * If the url is not valid, the try-catch condition will catch the exceptions and return false,
    * otherwise true will be returned.
-   * 
+   *
    * @param url
    * @return
    */
@@ -215,7 +215,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
       return gson.fromJson(json, User.class);
     }
   }
-  
+
   public void onLoginSuccess(String username, String session) {
     UserSessionContainer.instance.setSession(username, session);
 
@@ -227,7 +227,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
 
     ZeppelinhubUtils.userLoginRoutine(username);
   }
-  
+
   @Override
   public void onLogout(PrincipalCollection principals) {
     ZeppelinhubUtils.userLogoutRoutine((String) principals.getPrimaryPrincipal());

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -30,7 +30,6 @@ import java.util.regex.Pattern;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.vfs2.FileSystemException;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
 import org.apache.zeppelin.display.*;
@@ -1052,7 +1051,7 @@ public class NotebookServer extends WebSocketServlet
       note.persist(subject);
       addConnectionToNote(note.getId(), (NotebookSocket) conn);
       conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
-    } catch (FileSystemException e) {
+    } catch (IOException e) {
       LOG.error("Exception from createNote", e);
       conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
           "Oops! There is something wrong with the notebook file system. "
@@ -1846,7 +1845,7 @@ public class NotebookServer extends WebSocketServlet
     try {
       note.persist(p.getAuthenticationInfo());
       return true;
-    } catch (FileSystemException ex) {
+    } catch (IOException ex) {
       LOG.error("Exception from run", ex);
       conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
           "Oops! There is something wrong with the notebook file system. "

zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java

@@ -18,7 +18,9 @@ package org.apache.zeppelin.rest;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterNotFoundException;
 import org.apache.zeppelin.interpreter.InterpreterProperty;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterSetting;
@@ -367,15 +369,22 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
     Paragraph p3 = note.addNewParagraph(anonymous);
     p3.setText("%spark.pyspark print(z.get(\"var_1\"))");
 
+    // resources across interpreter processes (via DistributedResourcePool)
+    Paragraph p4 = note.addNewParagraph(anonymous);
+    p4.setText("%python print(z.get('var_1'))");
+
     note.run(p1.getId(), true);
     note.run(p2.getId(), true);
     note.run(p3.getId(), true);
+    note.run(p4.getId(), true);
 
     assertEquals(Status.FINISHED, p1.getStatus());
     assertEquals(Status.FINISHED, p2.getStatus());
     assertEquals("hello world\n", p2.getResult().message().get(0).getData());
     assertEquals(Status.FINISHED, p3.getStatus());
     assertEquals("hello world\n", p3.getResult().message().get(0).getData());
+    assertEquals(Status.FINISHED, p4.getStatus());
+    assertEquals("hello world\n", p4.getResult().message().get(0).getData());
   }
 
   @Test
@@ -502,8 +511,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
         "[('1', 'check_1'), ('2', 'check_2')], defaultChecked=['2'])\n" +
         "print(items[0])";
     p.setText(code);
-    note.run(p.getId());
-    waitForFinish(p);
+    note.run(p.getId(), true);
 
     assertEquals(Status.FINISHED, p.getStatus());
     Iterator<String> formIter = p.settings.getForms().keySet().iterator();
@@ -519,6 +527,51 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
     assertEquals("2", result[2]);
   }
 
+  @Test
+  public void testAngularObjects() throws IOException, InterpreterNotFoundException {
+    Note note = ZeppelinServer.notebook.createNote(anonymous);
+    Paragraph p1 = note.addNewParagraph(anonymous);
+
+    // add local angular object
+    p1.setText("%spark z.angularBind(\"name\", \"world\")");
+    note.run(p1.getId(), true);
+    assertEquals(Status.FINISHED, p1.getStatus());
+    List<AngularObject> angularObjects =
+        p1.getBindedInterpreter().getInterpreterGroup().getAngularObjectRegistry().getAll(note.getId(), null);
+    assertEquals(1, angularObjects.size());
+    assertEquals("name", angularObjects.get(0).getName());
+    assertEquals("world", angularObjects.get(0).get());
+
+    // remove local angular object
+    Paragraph p2 = note.addNewParagraph(anonymous);
+    p2.setText("%spark z.angularUnbind(\"name\")");
+    note.run(p2.getId(), true);
+    assertEquals(Status.FINISHED, p2.getStatus());
+    angularObjects =
+        p1.getBindedInterpreter().getInterpreterGroup().getAngularObjectRegistry().getAll(note.getId(), null);
+    assertEquals(0, angularObjects.size());
+
+    // add global angular object
+    Paragraph p3 = note.addNewParagraph(anonymous);
+    p3.setText("%spark z.angularBindGlobal(\"name2\", \"world2\")");
+    note.run(p3.getId(), true);
+    assertEquals(Status.FINISHED, p3.getStatus());
+    List<AngularObject> globalAngularObjects =
+        p3.getBindedInterpreter().getInterpreterGroup().getAngularObjectRegistry().getAll(null, null);
+    assertEquals(1, globalAngularObjects.size());
+    assertEquals("name2", globalAngularObjects.get(0).getName());
+    assertEquals("world2", globalAngularObjects.get(0).get());
+
+    // remove global angular object
+    Paragraph p4 = note.addNewParagraph(anonymous);
+    p4.setText("%spark z.angularUnbindGlobal(\"name2\")");
+    note.run(p4.getId(), true);
+    assertEquals(Status.FINISHED, p4.getStatus());
+    globalAngularObjects =
+        p4.getBindedInterpreter().getInterpreterGroup().getAngularObjectRegistry().getAll(note.getId(), null);
+    assertEquals(0, globalAngularObjects.size());
+  }
+
   @Test
   public void testConfInterpreter() throws IOException {
     ZeppelinServer.notebook.getInterpreterSettingManager().close();

zeppelin-server/src/test/resources/log4j.properties

@@ -41,5 +41,4 @@ log4j.logger.DataNucleus.Datastore=ERROR
 
 # Log all JDBC parameters
 log4j.logger.org.hibernate.type=ALL
-
-log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.hadoop=WARN

zeppelin-web/src/app/helium/helium.controller.js

@@ -150,7 +150,7 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
               console.log('Failed to save order');
               BootstrapDialog.show({
                 title: 'Error on saving order ',
-                message: data.message,
+                message: _.escape(data.message),
               });
             });
           return false;
@@ -244,8 +244,8 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
               confirm.close();
               console.log('Failed to enable package %o %o. %o', name, artifact, data);
               BootstrapDialog.show({
-                title: 'Error on enabling ' + name,
-                message: data.message,
+                title: 'Error on enabling ' + _.escape(name),
+                message: _.escape(data.message),
               });
             });
             return false;
@@ -261,7 +261,7 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
       closeByBackdrop: false,
       closeByKeyboard: false,
       title: '<div style="font-weight: 300;">Do you want to disable Helium Package?</div>',
-      message: artifact,
+      message: _.escape(artifact),
       callback: function(result) {
         if (result) {
           confirm.$modalFooter.find('button').addClass('disabled');
@@ -276,8 +276,8 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
             confirm.close();
             console.log('Failed to disable package %o. %o', name, data);
             BootstrapDialog.show({
-              title: 'Error on disabling ' + name,
-              message: data.message,
+              title: 'Error on disabling ' + _.escape(name),
+              message: _.escape(data.message),
             });
           });
           return false;

zeppelin-web/src/app/interpreter/interpreter.controller.js

@@ -508,7 +508,7 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou
       BootstrapDialog.alert({
         closable: true,
         title: 'Add interpreter',
-        message: 'Name ' + $scope.newInterpreterSetting.name + ' already exists',
+        message: 'Name ' + _.escape($scope.newInterpreterSetting.name) + ' already exists',
       });
       return;
     }
@@ -747,7 +747,7 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou
   $scope.showErrorMessage = function(setting) {
     BootstrapDialog.show({
       title: 'Error downloading dependencies',
-      message: setting.errorReason,
+      message: _.escape(setting.errorReason),
     });
   };
 
@@ -775,7 +775,7 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou
           window.open(res.data.body.url, '_blank');
         } else {
           BootstrapDialog.alert({
-            message: res.data.body.message,
+            message: _.escape(res.data.body.message),
           });
         }
       }).catch(function(res) {

zeppelin-web/src/app/jobmanager/job/job.component.js

@@ -94,7 +94,7 @@ class JobController {
     BootstrapDialog.alert({
       closable: true,
       title: title,
-      message: errorMessage,
+      message: _.escape(errorMessage),
     });
   }
 

zeppelin-web/src/app/notebook/notebook.controller.js

@@ -1010,7 +1010,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
       closeByBackdrop: false,
       closeByKeyboard: false,
       title: '',
-      message: 'Do you want to restart ' + interpreter.name + ' interpreter?',
+      message: 'Do you want to restart ' + _.escape(interpreter.name) + ' interpreter?',
       callback: function(result) {
         if (result) {
           let payload = {
@@ -1031,7 +1031,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
               console.log('Error %o %o', status, data.message);
               BootstrapDialog.show({
                 title: 'Error restart interpreter.',
-                message: data.message,
+                message: _.escape(data.message),
               });
             });
           return false;
@@ -1050,7 +1050,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
         closable: false,
         title: 'Setting Owners Permissions',
         message: 'Please fill the [Owners] field. If not, it will set as current user.\n\n' +
-          'Current user : [ ' + $rootScope.ticket.principal + ']',
+          'Current user : [ ' + _.escape($rootScope.ticket.principal) + ']',
         buttons: [
           {
             label: 'Set',
@@ -1083,9 +1083,13 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
         BootstrapDialog.alert({
           closable: true,
           title: 'Permissions Saved Successfully',
-          message: 'Owners : ' + $scope.permissions.owners + '\n\n' + 'Readers : ' +
-           $scope.permissions.readers + '\n\n' + 'Runners : ' + $scope.permissions.runners +
-           '\n\n' + 'Writers  : ' + $scope.permissions.writers,
+          message: 'Owners : ' + _.escape($scope.permissions.owners)
+          + '\n\n' +
+          'Readers : ' + _.escape($scope.permissions.readers) +
+          '\n\n' +
+          'Runners : ' + _.escape($scope.permissions.runners) +
+          '\n\n' +
+          'Writers  : ' + _.escape($scope.permissions.writers),
         });
         $scope.showPermissions = false;
       });
@@ -1097,7 +1101,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
         closeByBackdrop: false,
         closeByKeyboard: false,
         title: 'Insufficient privileges',
-        message: data.message,
+        message: _.escape(data.message),
         buttons: [
           {
             label: 'Login',

zeppelin-web/src/app/visualization/builtins/visualization-table.js

@@ -288,6 +288,51 @@ export default class TableVisualization extends Visualization {
 
     const config = this.config;
     const self = this; // for closure
+    const scope = this.getScope();
+    // set gridApi for this elem
+    const gridApiId = this.getGridApiId();
+    const gridOptions = this.createGridOptions(tableData, onRegisterApiCallback, config);
+
+    const onRegisterApiCallback = (gridApi) => {
+      scope[gridApiId] = gridApi;
+      // should restore state before registering APIs
+
+      // register callbacks for change evens
+      // should persist `self.config` instead `config` (closure issue)
+      gridApi.core.on.columnVisibilityChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.colMovable.on.columnPositionChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.core.on.sortChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.core.on.filterChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.grouping.on.aggregationChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.grouping.on.groupingChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.treeBase.on.rowCollapsed(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.treeBase.on.rowExpanded(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+      gridApi.colResizable.on.columnSizeChanged(scope, () => {
+        self.persistConfigWithGridState(self.config);
+      });
+
+      // pagination doesn't follow usual life-cycle in ui-grid v4.0.4
+      // gridApi.pagination.on.paginationChanged(scope, () => { self.persistConfigWithGridState(self.config) })
+      // TBD: do we need to propagate row selection?
+      // gridApi.selection.on.rowSelectionChanged(scope, () => { self.persistConfigWithGridState(self.config) })
+      // gridApi.selection.on.rowSelectionChangedBatch(scope, () => { self.persistConfigWithGridState(self.config) })
+    };
 
     if (!gridElem) {
       // create, compile and append grid elem
@@ -305,62 +350,14 @@ export default class TableVisualization extends Visualization {
               ui-grid-exporter></div>`);
 
       gridElem.css('height', this.targetEl.height() - 10);
-      const scope = this.getScope();
       gridElem = this._compile(gridElem)(scope);
       this.targetEl.append(gridElem);
-
-      // set gridOptions for this elem
-      const gridOptions = this.createGridOptions(tableData, onRegisterApiCallback, config);
       this.setDynamicGridOptions(gridOptions, config);
       this.addColumnMenus(gridOptions);
       scope[gridElemId] = gridOptions;
-
-      // set gridApi for this elem
-      const gridApiId = this.getGridApiId();
-      const onRegisterApiCallback = (gridApi) => {
-        scope[gridApiId] = gridApi;
-        // should restore state before registering APIs
-
-        // register callbacks for change evens
-        // should persist `self.config` instead `config` (closure issue)
-        gridApi.core.on.columnVisibilityChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.colMovable.on.columnPositionChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.core.on.sortChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.core.on.filterChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.grouping.on.aggregationChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.grouping.on.groupingChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.treeBase.on.rowCollapsed(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.treeBase.on.rowExpanded(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-        gridApi.colResizable.on.columnSizeChanged(scope, () => {
-          self.persistConfigWithGridState(self.config);
-        });
-
-        // pagination doesn't follow usual life-cycle in ui-grid v4.0.4
-        // gridApi.pagination.on.paginationChanged(scope, () => { self.persistConfigWithGridState(self.config) })
-        // TBD: do we need to propagate row selection?
-        // gridApi.selection.on.rowSelectionChanged(scope, () => { self.persistConfigWithGridState(self.config) })
-        // gridApi.selection.on.rowSelectionChangedBatch(scope, () => { self.persistConfigWithGridState(self.config) })
-      };
       gridOptions.onRegisterApi = onRegisterApiCallback;
     } else {
-      // don't need to update gridOptions.data since it's synchronized by paragraph execution
-      const gridOptions = this.getGridOptions();
+      scope[gridElemId] = gridOptions;
       this.setDynamicGridOptions(gridOptions, config);
       this.refreshGrid();
     }

zeppelin-web/src/components/note-action/note-action.service.js

@@ -139,7 +139,7 @@ function noteActionService(websocketMsgSrv, $location, noteRenameService, noteLi
             type: BootstrapDialog.TYPE_WARNING,
             closable: true,
             title: 'WARNING! The folder will be MERGED',
-            message: 'The folder will be merged into <strong>' + newFolderId + '</strong>. Are you sure?',
+            message: 'The folder will be merged into <strong>' + _.escape(newFolderId) + '</strong>. Are you sure?',
             callback: function(result) {
               if (result) {
                 websocketMsgSrv.renameFolder(folderId, newFolderId);

zeppelin-web/src/components/websocket/websocket-event.factory.js

@@ -150,7 +150,7 @@ function WebsocketEventFactory($rootScope, $websocket, $location, baseUrlSrv) {
         closeByBackdrop: false,
         closeByKeyboard: false,
         title: 'Details',
-        message: data.info.toString(),
+        message: _.escape(data.info.toString()),
         buttons: [{
           // close all the dialogs when there are error on running all paragraphs
           label: 'Close',

zeppelin-zengine/pom.xml

@@ -38,18 +38,13 @@
     <!--library versions-->
     <hadoop.version>2.7.3</hadoop.version>
     <commons.lang3.version>3.4</commons.lang3.version>
-    <commons.vfs2.version>2.0</commons.vfs2.version>
-    <gcs.storage.version>1.14.0</gcs.storage.version>
-    <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version>
-    <adl.sdk.version>2.1.4</adl.sdk.version>
     <jackrabbit.webdav.version>1.5.2</jackrabbit.webdav.version>
     <quartz.scheduler.version>2.2.1</quartz.scheduler.version>
     <lucene.version>5.3.1</lucene.version>
     <org.reflections.version>0.9.8</org.reflections.version>
     <xml.apis.version>1.4.01</xml.apis.version>
-    <eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version>
     <frontend.maven.plugin.version>1.3</frontend.maven.plugin.version>
-
+    <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version>
     <!--test library versions-->
     <google.truth.version>0.27</google.truth.version>
     <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version>
@@ -83,15 +78,8 @@
     </dependency>
 
     <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-vfs2</artifactId>
-      <version>${commons.vfs2.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.codehaus.plexus</groupId>
-          <artifactId>plexus-utils</artifactId>
-        </exclusion>
-      </exclusions>
+      <groupId>joda-time</groupId>
+      <artifactId>joda-time</artifactId>
     </dependency>
 
     <dependency>
@@ -116,86 +104,6 @@
       <artifactId>httpasyncclient</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.cloud</groupId>
-      <artifactId>google-cloud-storage</artifactId>
-      <version>${gcs.storage.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.google.code.findbugs</groupId>
-          <artifactId>jsr305</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.protobuf</groupId>
-          <artifactId>protobuf-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.guava</groupId>
-          <artifactId>guava</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.api</groupId>
-          <artifactId>api-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.http-client</groupId>
-          <artifactId>google-http-client-jackson2</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.http-client</groupId>
-          <artifactId>google-http-client</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.api</groupId>
-      <artifactId>api-common</artifactId>
-      <version>1.2.0</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.google.guava</groupId>
-          <artifactId>guava</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.code.findbugs</groupId>
-          <artifactId>jsr305</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.http-client</groupId>
-      <artifactId>google-http-client-jackson2</artifactId>
-      <version>1.23.0</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.fasterxml.jackson.core</groupId>
-          <artifactId>jackson-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.code.findbugs</groupId>
-          <artifactId>jsr305</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk-s3</artifactId>
-      <version>${aws.sdk.s3.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>joda-time</groupId>
-          <artifactId>joda-time</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-client</artifactId>
@@ -249,6 +157,12 @@
       <version>${lucene.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-s3</artifactId>
+      <version>${aws.sdk.s3.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>org.reflections</groupId>
       <artifactId>reflections</artifactId>
@@ -265,12 +179,6 @@
       </exclusions>
     </dependency>
 
-    <dependency>
-      <groupId>org.eclipse.jgit</groupId>
-      <artifactId>org.eclipse.jgit</artifactId>
-      <version>${eclipse.jgit.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>com.github.eirslett</groupId>
       <artifactId>frontend-maven-plugin</artifactId>
@@ -303,23 +211,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.cloud</groupId>
-      <artifactId>google-cloud-nio</artifactId>
-      <version>${google.testing.nio.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>com.google.code.findbugs</groupId>
-          <artifactId>jsr305</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.guava</groupId>
-          <artifactId>guava</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-
     <dependency>
       <groupId>com.google.truth</groupId>
       <artifactId>truth</artifactId>
@@ -366,12 +257,6 @@
       <version>1.5</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.mongodb</groupId>
-      <artifactId>mongo-java-driver</artifactId>
-      <version>3.4.1</version>
-    </dependency>
-
   </dependencies>
 
   <build>
@@ -745,73 +630,6 @@
             </exclusion>
           </exclusions>
         </dependency>
-
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-azure</artifactId>
-          <version>${hadoop.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.google.guava</groupId>
-              <artifactId>guava</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.commons</groupId>
-              <artifactId>commons-lang3</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.jcraf</groupId>
-              <artifactId>jsch</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.commons</groupId>
-              <artifactId>commons-compress</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>com.microsoft.azure</groupId>
-          <artifactId>azure-data-lake-store-sdk</artifactId>
-          <version>${adl.sdk.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-aws</artifactId>
-          <version>${hadoop.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.amazonaws</groupId>
-              <artifactId>aws-java-sdk</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-annotations</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-databind</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>joda-time</groupId>
-              <artifactId>joda-time</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        
       </dependencies>
     </profile>
 
@@ -837,104 +655,6 @@
           <version>${hadoop.version}</version>
           <scope>test</scope>
         </dependency>
-
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-azure</artifactId>
-          <version>${hadoop.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.google.guava</groupId>
-              <artifactId>guava</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.jcraft</groupId>
-              <artifactId>jsch</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.commons</groupId>
-              <artifactId>commons-compress</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.codehaus.jackson</groupId>
-              <artifactId>jackson-mapper-asl</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.nimbusds</groupId>
-              <artifactId>nimbus-jose-jwt</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.apache.zookeeper</groupId>
-              <artifactId>zookeeper</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.eclipse.jetty</groupId>
-              <artifactId>jetty-server</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.eclipse.jetty</groupId>
-              <artifactId>jetty-servlet</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.codehaus.jackson</groupId>
-              <artifactId>jackson-core-asl</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-databind</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>org.eclipse.jetty</groupId>
-              <artifactId>jetty-util</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.sun.jersey</groupId>
-              <artifactId>jersey-core</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-azure-datalake</artifactId>
-          <version>${hadoop.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-aws</artifactId>
-          <version>${hadoop.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.amazonaws</groupId>
-              <artifactId>aws-java-sdk</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-annotations</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-core</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>com.fasterxml.jackson.core</groupId>
-              <artifactId>jackson-databind</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>joda-time</groupId>
-              <artifactId>joda-time</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
       </dependencies>
     </profile>
   </profiles>

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java

@@ -33,14 +33,17 @@ public class ConfInterpreter extends Interpreter {
 
   private static Logger LOGGER = LoggerFactory.getLogger(ConfInterpreter.class);
 
-  private String interpreterGroupId;
-  private InterpreterSetting interpreterSetting;
+  protected String sessionId;
+  protected String interpreterGroupId;
+  protected InterpreterSetting interpreterSetting;
 
 
   public ConfInterpreter(Properties properties,
+                         String sessionId,
                          String interpreterGroupId,
                          InterpreterSetting interpreterSetting) {
     super(properties);
+    this.sessionId = sessionId;
     this.interpreterGroupId = interpreterGroupId;
     this.interpreterSetting = interpreterSetting;
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java

@@ -689,7 +689,16 @@ public class InterpreterSetting {
       LOGGER.info("Interpreter {} created for user: {}, sessionId: {}",
           interpreter.getClassName(), user, sessionId);
     }
-    interpreters.add(new ConfInterpreter(intpProperties, interpreterGroupId, this));
+
+    // TODO(zjffdu) this kind of hardcode is ugly. For now SessionConfInterpreter is used
+    // for livy, we could add new property in interpreter-setting.json when there's new interpreter
+    // require SessionConfInterpreter
+    if (group.equals("livy")) {
+      interpreters.add(
+          new SessionConfInterpreter(intpProperties, sessionId, interpreterGroupId, this));
+    } else {
+      interpreters.add(new ConfInterpreter(intpProperties, sessionId, interpreterGroupId, this));
+    }
     return interpreters;
   }
 
@@ -751,7 +760,11 @@ public class InterpreterSetting {
     //TODO(zjffdu) It requires user can not create interpreter with name `conf`,
     // conf is a reserved word of interpreter name
     if (replName.equals("conf")) {
-      return ConfInterpreter.class.getName();
+      if (group.equals("livy")) {
+        return SessionConfInterpreter.class.getName();
+      } else {
+        return ConfInterpreter.class.getName();
+      }
     }
     return null;
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java

@@ -34,8 +34,6 @@ import org.apache.zeppelin.dep.DependencyResolver;
 import org.apache.zeppelin.display.AngularObjectRegistryListener;
 import org.apache.zeppelin.helium.ApplicationEventListener;
 import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
-import org.apache.zeppelin.interpreter.recovery.FileSystemRecoveryStorage;
-import org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage;
 import org.apache.zeppelin.interpreter.recovery.RecoveryStorage;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
 import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
@@ -45,10 +43,8 @@ import org.apache.zeppelin.resource.ResourcePool;
 import org.apache.zeppelin.resource.ResourceSet;
 import org.apache.zeppelin.util.ReflectionUtils;
 import org.apache.zeppelin.storage.ConfigStorage;
-import org.apache.zeppelin.storage.FileSystemConfigStorage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonatype.aether.RepositoryException;
 import org.sonatype.aether.repository.Proxy;
 import org.sonatype.aether.repository.RemoteRepository;
 import org.sonatype.aether.repository.Authentication;