updated code to master branch

.travis.yml

@@ -22,17 +22,24 @@ before_install:
   - "sh -e /etc/init.d/xvfb start"
 
 install:
-  - mvn package -DskipTests -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B
+  - mvn package -DskipTests -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
 
 before_script:
   -
 
 script:
+ # spark 1.6
+  - mvn package -Pbuild-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
+  - ./testing/startSparkCluster.sh 1.6.0 2.3
+  - echo "export SPARK_HOME=`pwd`/spark-1.6.0-bin-hadoop2.3" > conf/zeppelin-env.sh
+  - mvn verify -Pusing-packaged-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
+  - ./testing/stopSparkCluster.sh 1.6.0 2.3
  # spark 1.5
-  - mvn package -Pbuild-distr -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B
+  - rm -rf `pwd`/interpreter/spark
+  - mvn package -DskipTests -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
   - ./testing/startSparkCluster.sh 1.5.2 2.3
   - echo "export SPARK_HOME=`pwd`/spark-1.5.2-bin-hadoop2.3" > conf/zeppelin-env.sh
-  - mvn verify -Pusing-packaged-distr -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B
+  - mvn package -Pspark-1.5 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
   - ./testing/stopSparkCluster.sh 1.5.2 2.3
  # spark 1.4
   - rm -rf `pwd`/interpreter/spark

DEPLOY.md

@@ -1,55 +0,0 @@
-Publish Zeppelin Distribution Package
-------
-
-A. Build package
-
-Following command will create package zeppelin-VERSION.tar.gz under _zeppelin-distribution/target_ directory.
-
-      mvn clean package -P build-distr
-
-
-B. Upload to S3 bucket ~~web server~~
-
-     ~~scp zeppelin-distribution/target/zeppelin-VERSION.tar.gz root@www.nflabs.com:/var/www/html/pub/zeppelin/~~
-     mvn package -P publish-distr
-
-
-C. Edit www.zeppelin-project.org
-
-Edit download page to have link to new release.
-
-
-Publish javadoc
--------
-
-Generate javadoc with following command
-
-    mvn javadoc:javadoc
-    mv "zeppelin-zengine/target/site/apidocs" "ZEPPELIN_HOMEPAGE/docs/zengine-api/VERSION"
-
-and publish the web.
-
-
-Publish Maven artifact
-------------
-
-**Publish to snapshot repository**
-
-     mvn -DperformRelease=true deploy
-
-
-**Publish to release repository**
-
-    mvn -DperformRelease=true release:clean
-    mvn -DperformRelease=true release:prepare
-    mvn -DperformRelease=true release:perform
-
-Artifact is now in staging repository.
-Connect https://oss.sonatype.org/ , select staging repository and click "close" -> "release" will finally release it.
-
-
-**Reference**
-
-https://docs.sonatype.org/display/Repository/How+To+Generate+PGP+Signatures+With+Maven
-
-https://docs.sonatype.org/display/Repository/Sonatype+OSS+Maven+Repository+Usage+Guide#SonatypeOSSMavenRepositoryUsageGuide-1a.TermsofServiceRepository%3ACentralRepositoryTermsofServiceforSubmitters

README.md

@@ -67,6 +67,7 @@ Set spark major version
 Available profiles are
 
 ```
+-Pspark-1.6
 -Pspark-1.5
 -Pspark-1.4
 -Pspark-1.3
@@ -134,13 +135,13 @@ Here're some examples:
 
 ```
 # basic build
-mvn clean package -Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark
+mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
 
 # spark-cassandra integration
 mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests
 
 # with CDH
-mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -Pvendor-repo -DskipTests
+mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests
 
 # with MapR
 mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
@@ -153,6 +154,11 @@ mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
 mvn clean package -Dignite.version=1.1.0-incubating -DskipTests
 ```
 
+#### Scalding Interpreter
+
+```
+mvn clean package -Pscalding -DskipTests
+```
 
 ### Configure
 If you wish to configure Zeppelin option (like port number), configure the following files:

SECURITY-README.md

@@ -0,0 +1,43 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Shiro Authentication
+To connect to Zeppelin, users will be asked to enter their credentials. Once logged, a user has access to all notes including other users notes.
+This a a first step toward full security as implemented by this pull request (https://github.com/apache/incubator-zeppelin/pull/53).
+
+# Security setup
+1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authcBasic" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections.  
+2. Secure the Websocket channel : Set to property "zeppelin.anonymous.allowed" to "false" in the file conf/zeppelin-site.xml. You can start by renaming conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
+3. Start Zeppelin : bin/zeppelin.sh
+4. point your browser to http://localhost:8080
+5. Login using one of the user/password combinations defined in the conf/shiro.ini file.
+
+# Implementation notes
+## Vocabulary
+username, owner and principal are used interchangeably to designate the currently authenticated user
+## What are we securing ?
+Zeppelin is basically a web application that spawn remote interpreters to run commands and return HTML fragments to be displayed on the user browser.
+The scope of this PR is to require credentials to access Zeppelin. To achieve this, we use Apache Shiro. 
+## HTTP Endpoint security
+Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info).
+## Websocket security
+Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach:
+1. The browser on startup requests a ticket through HTTP
+2. The Apache Shiro Servlet filter handles the user auth
+3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
+
+All websockets communications require the username and ticket  to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above).
+
+
+ 

bin/interpreter.sh

@@ -81,8 +81,11 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
     # This will evantually passes SPARK_APP_JAR to classpath of SparkIMain
     ZEPPELIN_CLASSPATH=${SPARK_APP_JAR}
 
+    pattern="$SPARK_HOME/python/lib/py4j-*-src.zip"
+    py4j=($pattern)
+    # pick the first match py4j zip - there should only be one
     export PYTHONPATH="$SPARK_HOME/python/:$PYTHONPATH"
-    export PYTHONPATH="$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH"
+    export PYTHONPATH="${py4j[0]}:$PYTHONPATH"
   else
     # add Hadoop jars into classpath
     if [[ -n "${HADOOP_HOME}" ]]; then
@@ -95,7 +98,11 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
     fi
 
     addJarInDir "${INTERPRETER_DIR}/dep"
-    PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip"
+
+    pattern="${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-*-src.zip"
+    py4j=($pattern)
+    # pick the first match py4j zip - there should only be one
+    PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${py4j[0]}"
 
     if [[ -z "${PYTHONPATH}" ]]; then
       export PYTHONPATH="${PYSPARKPATH}"

conf/shiro.ini

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[users]
+# List of users with their password allowed to access Zeppelin.
+# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
+admin = password1
+user1 = password2
+user2 = password3
+
+
+[urls]
+
+# anon means the access is anonymous.
+# authcBasic means Basic Auth Security
+# To enfore security, comment the line below and uncomment the next one
+/** = anon
+#/** = authcBasic
+

conf/zeppelin-site.xml.template

@@ -105,7 +105,7 @@
 
 <property>
   <name>zeppelin.interpreters</name>
-  <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter</value>
+  <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter</value>
   <description>Comma separated interpreter configurations. First interpreter become a default</description>
 </property>
 
@@ -180,5 +180,11 @@
   <description>Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173</description>
 </property>
 
+<property>
+  <name>zeppelin.anonymous.allowed</name>
+  <value>true</value>
+  <description>Anonymous user allowed by default</description>
+</property>
+
 </configuration>
 

docs/_includes/themes/zeppelin/_navigation.html

@@ -36,6 +36,8 @@
             <li>
               <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
               <ul class="dropdown-menu">
+                <li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
+                <li role="separator" class="divider"></li>
                 <li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
@@ -45,6 +47,7 @@
                 <li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
+                <li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
                 <li><a href="{{BASE_PATH}}/pleasecontribute.html">Shell</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
                 <li><a href="{{BASE_PATH}}/pleasecontribute.html">Tajo</a></li>

docs/docs.md

@@ -1,72 +0,0 @@
----
-layout: page
-title: "Docs"
-description: ""
-group: nav-right
----
-<!--
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-{% include JB/setup %}
-
-### Install
-
-* [Install](./install/install.html)
-* [YARN Install](./install/yarn_install.html)
-* [Virtual Machine Install](./install/virtual_machine.html)
-
-### Tutorial
-
-* [Tutorial](./tutorial/tutorial.html)
-
-### Interpreter
-
-**[Interpreters in zeppelin](manual/interpreters.html)**
-
-* [cassandra](./interpreter/cassandra.html)
-* [flink](./interpreter/flink.html)
-* [geode](./interpreter/geode.html)
-* [hive](./interpreter/hive.html)
-* [ignite](./interpreter/ignite.html)
-* [lens](./interpreter/lens.html)
-* [md](./interpreter/markdown.html)
-* [postgresql, hawq](./interpreter/postgresql.html)
-* [sh](./pleasecontribute.html)
-* [spark](./interpreter/spark.html)
-* [tajo](./pleasecontribute.html)
-* [elasticsearch](./interpreter/elasticsearch.html)
-
-### Storage
-* [S3 Storage](./storage/storage.html)
-
-### Display System
-
-* [text](./displaysystem/display.html)
-* [html](./displaysystem/display.html#html)
-* [table](./displaysystem/table.html)
-* [angular](./displaysystem/angular.html) (Beta)
-
-### Manual
-
-* [Dynamic Form](./manual/dynamicform.html)
-* [Notebook as Homepage](./manual/notebookashomepage.html)
-
-### REST API
- * [Interpreter API](./rest-api/rest-interpreter.html)
- * [Notebook API](./rest-api/rest-notebook.html)
-
-### Development
-
-* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
-* [How to contribute (code)](./development/howtocontribute.html)
-* [How to contribute (website)](./development/howtocontributewebsite.html)

docs/install/install.md

@@ -176,12 +176,6 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
     <td>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</td>
     <td>Comma separated list of notebook storage</td>
   </tr>
-  <tr>
-    <td>ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE</td>
-    <td>zeppelin.notebook.reloadAllNotesFromStorage</td>
-    <td>false</td>
-    <td>Notebook list and contents will be always loaded from repository if set true. If set false, modified notebooks or new notebooks added on file system level won't be reflected on Zeppelin till user restarts Zeppelin.</td>
-  </tr>
   <tr>
     <td>ZEPPELIN_INTERPRETERS</td>
     <td>zeppelin.interpreters</td>

docs/interpreter/elasticsearch.md

@@ -110,6 +110,7 @@ With the `search` command, you can send a search query to Elasticsearch. There a
   * This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }` 
   * See [Elasticsearch query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) for more details about the content of such a query.
 
+
 ```bash
 | %elasticsearch
 | search /index1,index2,.../type1,type2,...  <JSON document containing the query or query_string elements>
@@ -124,6 +125,9 @@ If you want to modify the size of the result set, you can add a line that is set
 ```
 
 
+> A search query can also contain [aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html). If there is at least one aggregation, the result of the first aggregation is shown, otherwise, you get the search hits.
+
+
 Examples:
 
   * With a JSON query:
@@ -134,6 +138,15 @@ Examples:
 |
 | %elasticsearch
 | search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
+|
+| %elasticsearch
+| search /logs { "aggs": {
+|   "content_length_stats": {
+|     "extended_stats": {
+|       "field": "content_length"
+|     }
+|   }
+| } } 
   ```
 
   * With query_string elements:
@@ -159,16 +172,17 @@ Suppose we have a JSON document:
     "url": "/zeppelin/4cd001cd-c517-4fa9-b8e5-a06b8f4056c4",
     "headers": [ "Accept: *.*", "Host: apache.org"]
   },
-  "status": "403"
+  "status": "403",
+  "content_length": 1234
 }
 ```
 
 The data will be flattened like this:
 
 
-date | request.headers[0] | request.headers[1] | request.method | request.url | status
------|--------------------|--------------------|----------------|-------------|-------
-2015-12-08T21:03:13.588Z | Accept: \*.\* | Host: apache.org | GET | /zeppelin/4cd001cd-c517-4fa9-b8e5-a06b8f4056c4 | 403
+content_length | date | request.headers[0] | request.headers[1] | request.method | request.url | status
+---------------|------|--------------------|--------------------|----------------|-------------|-------
+1234 | 2015-12-08T21:03:13.588Z | Accept: \*.\* | Host: apache.org | GET | /zeppelin/4cd001cd-c517-4fa9-b8e5-a06b8f4056c4 | 403
 
 
 Examples:
@@ -185,6 +199,12 @@ Examples:
 * With a query string:
 ![Elasticsearch - Search with query string](../assets/themes/zeppelin/img/docs-img/elasticsearch-query-string.png)
 
+* With a query containing a multi-value metric aggregation:
+![Elasticsearch - Search with aggregation (multi-value metric)](../assets/themes/zeppelin/img/docs-img/elasticsearch-agg-multi-value-metric.png)
+
+* With a query containing a multi-bucket aggregation:
+![Elasticsearch - Search with aggregation (multi-bucket)](../assets/themes/zeppelin/img/docs-img/elasticsearch-agg-multi-bucket-pie.png)
+
 
 #### count
 With the `count` command, you can count documents available in some indices and types. You can also provide a query.

docs/interpreter/scalding.md

@@ -0,0 +1,78 @@
+---
+layout: page
+title: "Scalding Interpreter"
+description: ""
+group: manual
+---
+{% include JB/setup %}
+
+
+## Scalding Interpreter for Apache Zeppelin
+[Scalding](https://github.com/twitter/scalding) is an open source Scala library for writing MapReduce jobs.
+
+### Building the Scalding Interpreter
+You have to first build the Scalding interpreter by enable the **scalding** profile as follows:
+
+```
+mvn clean package -Pscalding -DskipTests
+```
+
+### Enabling the Scalding Interpreter
+
+In a notebook, to enable the **Scalding** interpreter, click on the **Gear** icon,select **Scalding**, and hit **Save**.
+ 
+ <center>
+ ![Interpreter Binding](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterBinding.png)
+
+ ![Interpreter Selection](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterSelection.png)
+ </center>
+
+### Configuring the Interpreter
+Zeppelin comes with a pre-configured Scalding interpreter in local mode, so you do not need to install anything.
+
+### Testing the Interpreter
+
+In example, by using the [Alice in Wonderland](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) tutorial, we will count words (of course!), and plot a graph of the top 10 words in the book.
+
+```
+%scalding
+
+import scala.io.Source
+
+// Get the Alice in Wonderland book from gutenberg.org:
+val alice = Source.fromURL("http://www.gutenberg.org/files/11/11.txt").getLines
+val aliceLineNum = alice.zipWithIndex.toList
+val alicePipe = TypedPipe.from(aliceLineNum)
+
+// Now get a list of words for the book:
+val aliceWords = alicePipe.flatMap { case (text, _) => text.split("\\s+").toList }
+
+// Now lets add a count for each word:
+val aliceWithCount = aliceWords.filterNot(_.equals("")).map { word => (word, 1L) }
+
+// let's sum them for each word:
+val wordCount = aliceWithCount.group.sum
+
+print ("Here are the top 10 words\n")
+val top10 = wordCount
+  .groupAll
+  .sortBy { case (word, count) => -count }
+  .take(10)
+top10.dump
+
+```
+```
+%scalding
+
+val table = "words\t count\n" + top10.toIterator.map{case (k, (word, count)) => s"$word\t$count"}.mkString("\n")
+print("%table " + table)
+
+```
+
+If you click on the icon for the pie chart, you should be able to see a chart like this:
+![Scalding - Pie - Chart](../assets/themes/zeppelin/img/docs-img/scalding-pie.png)
+
+### Current Status & Future Work
+The current implementation of the Scalding interpreter does not support canceling jobs, or fine-grained progress updates. 
+
+The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).

docs/rest-api/rest-notebook.md

@@ -381,7 +381,7 @@ limitations under the License.
     </tr>
     <tr>
       <td> sample JSON response </td>
-      <td><pre>{"status":"OK","body":[{"id":"20151121-212654_766735423","status":"FINISHED","finished":"Tue Nov 24 14:21:40 KST 2015","started":"Tue Nov 24 14:21:39 KST 2015"},{"id":"20151121-212657_730976687","status":"FINISHED","finished":"Tue Nov 24 14:21:40 KST 2015","started":"Tue Nov 24 14:21:40 KST 2015"}]}</pre></td>
+      <td><pre>{"status":"OK","body":[{"id":"20151121-212654_766735423","status":"FINISHED","finished":"Tue Nov 24 14:21:40 KST 2015","started":"Tue Nov 24 14:21:39 KST 2015"},{"progress":"1","id":"20151121-212657_730976687","status":"RUNNING","finished":"Tue Nov 24 14:21:35 KST 2015","started":"Tue Nov 24 14:21:40 KST 2015"}]}</pre></td>
     </tr>
   </table>
   
@@ -586,3 +586,190 @@ limitations under the License.
     </tr>
   </table>
   
+<br/>
+
+
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <th>Create paragraph</th>
+      <th></th>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>This ```POST``` method create a new paragraph using JSON payload.
+          The body field of the returned JSON contain the new paragraph id.
+      </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[notebookId]/paragraph```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>201</td>
+    </tr>
+    <tr>
+      <td> Fail code</td>
+      <td> 500 </td>
+    </tr>
+    <tr>
+      <td> sample JSON input (add to the last) </td>
+      <td><pre>
+  { 
+    "title": "Paragraph insert revised", 
+    "text": "%spark\nprintln(\"Paragraph insert revised\")" 
+  }</pre></td>
+    </tr>
+    <tr>
+      <td> sample JSON input (add to specific index) </td>
+      <td><pre>
+  { 
+    "title": "Paragraph insert revised", 
+    "text": "%spark\nprintln(\"Paragraph insert revised\")",
+    "index": 0
+  }
+      </pre></td>
+    </tr>
+    <tr>
+      <td> sample JSON response </td>
+      <td><pre>{"status": "CREATED","message": "","body": "20151218-100330_1754029574"}</pre></td>
+    </tr>
+  </table>
+  
+<br/>
+
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <th>Get paragraph</th>
+      <th></th>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>This ```GET``` method retrieves an existing paragraph's information using the given id.
+          The body field of the returned JSON contain information about paragraph.
+      </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[notebookId]/paragraph/[paragraphId]```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>200</td>
+    </tr>
+    <tr>
+      <td> Fail code</td>
+      <td> 500 </td>
+    </tr>
+    <tr>
+      <td> sample JSON response </td>
+      <td><pre>
+{
+  "status": "OK",
+  "message": "",
+  "body": {
+    "title": "Paragraph2",
+    "text": "%spark\n\nprintln(\"it's paragraph2\")",
+    "dateUpdated": "Dec 18, 2015 7:33:54 AM",
+    "config": {
+      "colWidth": 12,
+      "graph": {
+        "mode": "table",
+        "height": 300,
+        "optionOpen": false,
+        "keys": [],
+        "values": [],
+        "groups": [],
+        "scatter": {}
+      },
+      "enabled": true,
+      "title": true,
+      "editorMode": "ace/mode/scala"
+    },
+    "settings": {
+      "params": {},
+      "forms": {}
+    },
+    "jobName": "paragraph_1450391574392_-1890856722",
+    "id": "20151218-073254_1105602047",
+    "result": {
+      "code": "SUCCESS",
+      "type": "TEXT",
+      "msg": "it's paragraph2\n"
+    },
+    "dateCreated": "Dec 18, 2015 7:32:54 AM",
+    "dateStarted": "Dec 18, 2015 7:33:55 AM",
+    "dateFinished": "Dec 18, 2015 7:33:55 AM",
+    "status": "FINISHED",
+    "progressUpdateIntervalMs": 500
+  }
+}
+      </pre></td>
+    </tr>
+  </table>
+  
+<br/>
+
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <th>Move paragraph</th>
+      <th></th>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>This ```POST``` method moves a paragraph to the specific index (order) from the notebook.
+      </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[notebookId]/paragraph/[paragraphId]/move/[newIndex]```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>200</td>
+    </tr>
+    <tr>
+      <td> Fail code</td>
+      <td> 500 </td>
+    </tr>
+    <tr>
+      <td> sample JSON response </td>
+      <td><pre>{"status":"OK","message":""}</pre></td>
+    </tr>
+  </table>
+  
+
+<br/>
+
+  <table class="table-configuration">
+    <col width="200">
+    <tr>
+      <th>Delete paragraph</th>
+      <th></th>
+    </tr>
+    <tr>
+      <td>Description</td>
+      <td>This ```DELETE``` method deletes a paragraph by the given notebook and paragraph id.
+      </td>
+    </tr>
+    <tr>
+      <td>URL</td>
+      <td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[notebookId]/paragraph/[paragraphId]```</td>
+    </tr>
+    <tr>
+      <td>Success code</td>
+      <td>200</td>
+    </tr>
+    <tr>
+      <td> Fail code</td>
+      <td> 500 </td>
+    </tr>
+    <tr>
+      <td> sample JSON response </td>
+      <td><pre>{"status":"OK","message":""}</pre></td>
+    </tr>
+  </table>
+  

elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java

@@ -17,17 +17,10 @@
 
 package org.apache.zeppelin.elasticsearch;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TreeSet;
-
+import com.github.wnameless.json.flattener.JsonFlattener;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -43,16 +36,21 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.github.wnameless.json.flattener.JsonFlattener;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonParseException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.*;
 
 
 /**
@@ -313,7 +311,8 @@ public class ElasticsearchInterpreter extends Interpreter {
    * Processes a "search" request.
    * 
    * @param urlItems Items of the URL
-   * @param data May contains the limit and the JSON of the request
+   * @param data May contains the JSON of the request
+   * @param size Limit of result set
    * @return Result of the search request, it contains a tab-formatted string of the matching hits
    */
   private InterpreterResult processSearch(String[] urlItems, String data, int size) {
@@ -325,10 +324,7 @@ public class ElasticsearchInterpreter extends Interpreter {
         
     final SearchResponse response = searchData(urlItems, data, size);
 
-    return new InterpreterResult(
-      InterpreterResult.Code.SUCCESS,
-      InterpreterResult.Type.TABLE,
-      buildResponseMessage(response.getHits().getHits()));
+    return buildResponseMessage(response);
   }
 
   /**
@@ -419,7 +415,39 @@ public class ElasticsearchInterpreter extends Interpreter {
     return response;
   }
 
-  private String buildResponseMessage(SearchHit[] hits) {
+  private InterpreterResult buildAggResponseMessage(Aggregations aggregations) {
+
+    // Only the result of the first aggregation is returned
+    //
+    final Aggregation agg = aggregations.asList().get(0);
+    InterpreterResult.Type resType = InterpreterResult.Type.TEXT;
+    String resMsg = "";
+
+    if (agg instanceof InternalMetricsAggregation) {
+      resMsg = XContentHelper.toString((InternalMetricsAggregation) agg).toString();
+    }
+    else if (agg instanceof InternalSingleBucketAggregation) {
+      resMsg = XContentHelper.toString((InternalSingleBucketAggregation) agg).toString();
+    }
+    else if (agg instanceof InternalMultiBucketAggregation) {
+      final StringBuffer buffer = new StringBuffer("key\tdoc_count");
+
+      final InternalMultiBucketAggregation multiBucketAgg = (InternalMultiBucketAggregation) agg;
+      for (MultiBucketsAggregation.Bucket bucket : multiBucketAgg.getBuckets()) {
+        buffer.append("\n")
+          .append(bucket.getKeyAsString())
+          .append("\t")
+          .append(bucket.getDocCount());
+      }
+
+      resType = InterpreterResult.Type.TABLE;
+      resMsg = buffer.toString();
+    }
+
+    return new InterpreterResult(InterpreterResult.Code.SUCCESS, resType, resMsg);
+  }
+
+  private String buildSearchHitsResponseMessage(SearchHit[] hits) {
         
     if (hits == null || hits.length == 0) {
       return "";
@@ -462,4 +490,18 @@ public class ElasticsearchInterpreter extends Interpreter {
 
     return buffer.toString();
   }
+
+  private InterpreterResult buildResponseMessage(SearchResponse response) {
+
+    final Aggregations aggregations = response.getAggregations();
+
+    if (aggregations != null && aggregations.asList().size() > 0) {
+      return buildAggResponseMessage(aggregations);
+    }
+
+    return new InterpreterResult(
+      InterpreterResult.Code.SUCCESS,
+      InterpreterResult.Type.TABLE,
+      buildSearchHitsResponseMessage(response.getHits().getHits()));
+  }
 }

elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java

@@ -17,31 +17,27 @@
 
 package org.apache.zeppelin.elasticsearch;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Properties;
-import java.util.UUID;
-
 import org.apache.commons.lang.math.RandomUtils;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Properties;
+import java.util.UUID;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.junit.Assert.assertEquals;
+
 public class ElasticsearchInterpreterTest {
     
   private static Client elsClient;
@@ -49,7 +45,7 @@ public class ElasticsearchInterpreterTest {
   private static ElasticsearchInterpreter interpreter;
     
   private static final String[] METHODS = { "GET", "PUT", "DELETE", "POST" };
-  private static final String[] STATUS = { "200", "404", "500", "403" };
+  private static final int[] STATUS = { 200, 404, 500, 403 };
 
   private static final String ELS_CLUSTER_NAME = "zeppelin-elasticsearch-interpreter-test";
   private static final String ELS_HOST = "localhost";
@@ -71,6 +67,14 @@ public class ElasticsearchInterpreterTest {
 
     elsNode = NodeBuilder.nodeBuilder().settings(settings).node();
     elsClient = elsNode.client();
+
+    elsClient.admin().indices().prepareCreate("logs")
+      .addMapping("http", jsonBuilder()
+        .startObject().startObject("http").startObject("properties")
+          .startObject("content_length")
+            .field("type", "integer")
+          .endObject()
+        .endObject().endObject().endObject()).get();
         
     for (int i = 0; i < 50; i++) {
       elsClient.prepareIndex("logs", "http", "" + i)
@@ -84,6 +88,7 @@ public class ElasticsearchInterpreterTest {
               .field("headers", Arrays.asList("Accept: *.*", "Host: apache.org"))
             .endObject()
             .field("status", STATUS[RandomUtils.nextInt(STATUS.length)])
+            .field("content_length", RandomUtils.nextInt(2000))
           )
         .get();
     }
@@ -147,6 +152,31 @@ public class ElasticsearchInterpreterTest {
     res = interpreter.interpret("search /logs status:404", null);
     assertEquals(Code.SUCCESS, res.code());   
   }
+
+  @Test
+  public void testAgg() {
+
+    // Single-value metric
+    InterpreterResult res = interpreter.interpret("search /logs { \"aggs\" : { \"distinct_status_count\" : " +
+            " { \"cardinality\" : { \"field\" : \"status\" } } } }", null);
+    assertEquals(Code.SUCCESS, res.code());
+
+    // Multi-value metric
+    res = interpreter.interpret("search /logs { \"aggs\" : { \"content_length_stats\" : " +
+            " { \"extended_stats\" : { \"field\" : \"content_length\" } } } }", null);
+    assertEquals(Code.SUCCESS, res.code());
+
+    // Single bucket
+    res = interpreter.interpret("search /logs { \"aggs\" : { " +
+            " \"200_OK\" : { \"filter\" : { \"term\": { \"status\": \"200\" } }, " +
+            "   \"aggs\" : { \"avg_length\" : { \"avg\" : { \"field\" : \"content_length\" } } } } } }", null);
+    assertEquals(Code.SUCCESS, res.code());
+
+    // Multi-buckets
+    res = interpreter.interpret("search /logs { \"aggs\" : { \"status_count\" : " +
+            " { \"terms\" : { \"field\" : \"status\" } } } }", null);
+    assertEquals(Code.SUCCESS, res.code());
+  }
     
   @Test
   public void testIndex() {

ignite/pom.xml

@@ -33,7 +33,7 @@
   <url>http://zeppelin.incubator.apache.org</url>
 
   <properties>
-    <ignite.version>1.4.0</ignite.version>
+    <ignite.version>1.5.0.final</ignite.version>
     <ignite.scala.binary.version>2.10</ignite.scala.binary.version>
     <ignite.scala.version>2.10.4</ignite.scala.version>
   </properties>

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java

@@ -211,6 +211,7 @@ public class IgniteInterpreter extends Interpreter {
 
         initEx = null;
       } catch (Exception e) {
+        logger.error("Error in IgniteInterpreter while getIgnite: " , e);
         initEx = e;
       }
     }

ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java

@@ -154,6 +154,7 @@ public class IgniteSqlInterpreter extends Interpreter {
         }
       }
     } catch (Exception e) {
+      logger.error("Exception in IgniteSqlInterpreter while InterpreterResult interpret: ", e);
       return IgniteInterpreterUtils.buildErrorResult(e);
     } finally {
       curStmt = null;
@@ -169,6 +170,7 @@ public class IgniteSqlInterpreter extends Interpreter {
         curStmt.cancel();
       } catch (SQLException e) {
         // No-op.
+        logger.info("No-op while cancel in IgniteSqlInterpreter", e);
       } finally {
         curStmt = null;
       }

ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java

@@ -24,6 +24,7 @@ import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.zeppelin.interpreter.InterpreterContext;
@@ -59,6 +60,7 @@ public class IgniteSqlInterpreterTest {
     IgniteConfiguration cfg = new IgniteConfiguration();
     cfg.setDiscoverySpi(discoSpi);
     cfg.setPeerClassLoadingEnabled(true);
+    cfg.setMarshaller(new OptimizedMarshaller());
 
     cfg.setGridName("test");
 

lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java

@@ -128,7 +128,7 @@ public class LensInterpreter extends Interpreter {
       s_logger.info("LensInterpreter created");
     }
     catch (Exception e) {
-      e.printStackTrace();
+      s_logger.error(e.toString(), e);
       s_logger.error("unable to create lens interpreter", e);
     }
   }
@@ -375,6 +375,7 @@ public class LensInterpreter extends Interpreter {
         closeShell(s_paraToQH.get(context.getParagraphId()).getShell());
       } catch (Exception e) {
         // ignore
+        s_logger.info("Exception in LensInterpreter while cancel finally, ignore", e);
       }
       s_paraToQH.remove(context.getParagraphId());
       closeShell(shell);

lens/src/main/java/org/apache/zeppelin/lens/LensJLineShellComponent.java

@@ -17,6 +17,9 @@ package org.apache.zeppelin.lens;
 
 import java.util.Map;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.BeanFactoryUtils;
 import org.springframework.beans.factory.InitializingBean;
@@ -56,6 +59,8 @@ public class LensJLineShellComponent extends JLineShell
   private ExecutionStrategy executionStrategy = new LensSimpleExecutionStrategy();
   private SimpleParser parser = new SimpleParser();
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(LensJLineShellComponent.class);
+
   public SimpleParser getSimpleParser() {
     return parser;
   }
@@ -123,7 +128,7 @@ public class LensJLineShellComponent extends JLineShell
     try {
       shellThread.join();
     } catch (InterruptedException e) {
-      e.printStackTrace();
+      LOGGER.error(e.toString(), e);
     }
   }
 

markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java

@@ -29,16 +29,15 @@ import org.apache.zeppelin.interpreter.InterpreterUtils;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
 import org.markdown4j.Markdown4jProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Markdown interpreter for Zeppelin.
- *
- * @author Leemoonsoo
- * @author anthonycorbacho
- *
  */
 public class Markdown extends Interpreter {
   private Markdown4jProcessor md;
+  static final Logger LOGGER = LoggerFactory.getLogger(Markdown.class);
 
   static {
     Interpreter.register("md", Markdown.class.getName());
@@ -62,6 +61,7 @@ public class Markdown extends Interpreter {
     try {
       html = md.process(st);
     } catch (IOException | java.lang.RuntimeException e) {
+      LOGGER.error("Exception in Markdown while interpret ", e);
       return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
     }
     return new InterpreterResult(Code.SUCCESS, "%html " + html);

pom.xml

@@ -209,6 +209,18 @@
         <version>4.11</version>
         <scope>test</scope>
       </dependency>
+
+      <!-- Apache Shiro -->
+      <dependency>
+        <groupId>org.apache.shiro</groupId>
+        <artifactId>shiro-core</artifactId>
+        <version>1.2.3</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.shiro</groupId>
+        <artifactId>shiro-web</artifactId>
+        <version>1.2.3</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -628,6 +640,13 @@
       </modules>
     </profile>
 
+    <profile>
+      <id>scalding</id>
+      <modules>
+        <module>scalding</module>
+      </modules>
+    </profile>
+
     <profile>
       <id>build-distr</id>
       <activation>
@@ -688,7 +707,7 @@
         <plugins>
           <plugin>
             <artifactId>maven-gpg-plugin</artifactId>
-            <version>1.4</version>
+            <version>1.6</version>
             <executions>
               <execution>
                 <id>sign-artifacts</id>
@@ -703,15 +722,5 @@
       </build>
     </profile>
   </profiles>
-  <distributionManagement>
-    <site>
-      <id>Website</id>
-      <url>${site_url}</url>
-    </site>
-    <repository>
-      <id>${repoid}</id>
-      <name>${reponame}</name>
-      <url>${repourl}</url>
-    </repository>
-  </distributionManagement>
+
 </project>

postgresql/src/main/java/org/apache/zeppelin/postgresql/PostgreSqlInterpreter.java

@@ -297,6 +297,7 @@ public class PostgreSqlInterpreter extends Interpreter {
       try {
         currentStatement.cancel();
       } catch (SQLException ex) {
+        logger.error("SQLException in PostgreSqlInterpreter while cancel ", ex);
       } finally {
         currentStatement = null;
       }

scalding/pom.xml

@@ -0,0 +1,202 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>zeppelin</artifactId>
+    <groupId>org.apache.zeppelin</groupId>
+    <version>0.6.0-incubating-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <groupId>org.apache.zeppelin</groupId>
+  <artifactId>zeppelin-scalding</artifactId>
+  <packaging>jar</packaging>
+  <version>0.6.0-incubating-SNAPSHOT</version>
+  <name>Zeppelin: Scalding interpreter</name>
+  <url>http://zeppelin.incubator.apache.org</url>
+
+  <properties>
+    <scala.version>2.10.4</scala.version>
+    <hadoop.version>2.3.0</hadoop.version>
+    <scalding.version>0.15.1-RC13</scalding.version>
+  </properties>
+
+  <repositories>
+    <repository>
+      <id>conjars</id>
+      <name>Concurrent Maven Repo</name>
+      <url>http://conjars.org/repo</url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zeppelin-interpreter</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-exec</artifactId>
+      <version>1.3</version>
+    </dependency>
+    
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.twitter</groupId>
+      <artifactId>scalding-core_2.10</artifactId>
+      <version>${scalding.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.twitter</groupId>
+      <artifactId>scalding-repl_2.10</artifactId>
+      <version>${scalding.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-compiler</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-reflect</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+
+    <!-- Scalding REPL needs org.apache.hadoop.conf.Configuration even in local mode -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <version>2.7</version>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.3.1</version>            
+        <executions> 
+          <execution> 
+            <id>enforce</id> 
+            <phase>none</phase> 
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>2.8</version>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/../../interpreter/scalding</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+              <overWriteIfNewer>true</overWriteIfNewer>
+              <includeScope>runtime</includeScope>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-artifact</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/../../interpreter/scalding</outputDirectory>
+              <overWriteReleases>false</overWriteReleases>
+              <overWriteSnapshots>false</overWriteSnapshots>
+              <overWriteIfNewer>true</overWriteIfNewer>
+              <includeScope>runtime</includeScope>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>${project.artifactId}</artifactId>
+                  <version>${project.version}</version>
+                  <type>${project.packaging}</type>
+                </artifactItem>
+              </artifactItems>              
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- Plugin to compile Scala code -->
+      <plugin>
+        <groupId>org.scala-tools</groupId>
+        <artifactId>maven-scala-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>compile</id>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+          <execution>
+            <id>test-compile</id>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+            <phase>test-compile</phase>
+          </execution>
+          <execution>
+            <phase>process-resources</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java

@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.scalding;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import scala.Console;
+import scala.Some;
+import scala.None;
+import scala.tools.nsc.Settings;
+import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
+import scala.tools.nsc.settings.MutableSettings.PathSetting;
+
+/**
+ * Scalding interpreter for Zeppelin. Based off the Spark interpreter code.
+ *
+ */
+public class ScaldingInterpreter extends Interpreter {
+  Logger logger = LoggerFactory.getLogger(ScaldingInterpreter.class);
+
+  public static final List<String> NO_COMPLETION = 
+    Collections.unmodifiableList(new ArrayList<String>());
+
+  static {
+    Interpreter.register("scalding", ScaldingInterpreter.class.getName());
+  }
+
+  private ScaldingILoop interpreter;
+  private ByteArrayOutputStream out;
+  private Map<String, Object> binder;
+
+  public ScaldingInterpreter(Properties property) {
+    super(property);
+    out = new ByteArrayOutputStream();
+  }
+
+  @Override
+  public void open() {
+    URL[] urls = getClassloaderUrls();
+
+    // Very nice discussion about how scala compiler handle classpath
+    // https://groups.google.com/forum/#!topic/scala-user/MlVwo2xCCI0
+
+    /*
+     * > val env = new nsc.Settings(errLogger) > env.usejavacp.value = true > val p = new
+     * Interpreter(env) > p.setContextClassLoader > Alternatively you can set the class path through
+     * nsc.Settings.classpath.
+     *
+     * >> val settings = new Settings() >> settings.usejavacp.value = true >>
+     * settings.classpath.value += File.pathSeparator + >> System.getProperty("java.class.path") >>
+     * val in = new Interpreter(settings) { >> override protected def parentClassLoader =
+     * getClass.getClassLoader >> } >> in.setContextClassLoader()
+     */
+    Settings settings = new Settings();
+
+    // set classpath for scala compiler
+    PathSetting pathSettings = settings.classpath();
+    String classpath = "";
+    List<File> paths = currentClassPath();
+    for (File f : paths) {
+      if (classpath.length() > 0) {
+        classpath += File.pathSeparator;
+      }
+      classpath += f.getAbsolutePath();
+    }
+
+    if (urls != null) {
+      for (URL u : urls) {
+        if (classpath.length() > 0) {
+          classpath += File.pathSeparator;
+        }
+        classpath += u.getFile();
+      }
+    }
+
+    pathSettings.v_$eq(classpath);
+    settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
+
+
+    // set classloader for scala compiler
+    settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread()
+        .getContextClassLoader()));
+    BooleanSetting b = (BooleanSetting) settings.usejavacp();
+    b.v_$eq(true);
+    settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
+
+    /* Scalding interpreter */
+    PrintStream printStream = new PrintStream(out);
+    interpreter = new ScaldingILoop(null, new PrintWriter(out));
+    interpreter.settings_$eq(settings);
+    interpreter.createInterpreter();
+
+    interpreter.intp().
+      interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
+    binder = (Map<String, Object>) getValue("_binder");
+    binder.put("out", printStream);
+  }
+
+  private Object getValue(String name) {
+    Object ret = interpreter.intp().valueOfTerm(name);
+    if (ret instanceof None) {
+      return null;
+    } else if (ret instanceof Some) {
+      return ((Some) ret).get();
+    } else {
+      return ret;
+    }
+  }
+
+  private List<File> currentClassPath() {
+    List<File> paths = classPath(Thread.currentThread().getContextClassLoader());
+    String[] cps = System.getProperty("java.class.path").split(File.pathSeparator);
+    if (cps != null) {
+      for (String cp : cps) {
+        paths.add(new File(cp));
+      }
+    }
+    return paths;
+  }
+
+  private List<File> classPath(ClassLoader cl) {
+    List<File> paths = new LinkedList<File>();
+    if (cl == null) {
+      return paths;
+    }
+
+    if (cl instanceof URLClassLoader) {
+      URLClassLoader ucl = (URLClassLoader) cl;
+      URL[] urls = ucl.getURLs();
+      if (urls != null) {
+        for (URL url : urls) {
+          paths.add(new File(url.getFile()));
+        }
+      }
+    }
+    return paths;
+  }
+
+  @Override
+  public void close() {
+    interpreter.intp().close();
+  }
+
+
+  @Override
+  public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
+    logger.info("Running Scalding command '" + cmd + "'");
+
+    if (cmd == null || cmd.trim().length() == 0) {
+      return new InterpreterResult(Code.SUCCESS);
+    }
+    return interpret(cmd.split("\n"), contextInterpreter);
+  }
+
+  public InterpreterResult interpret(String[] lines, InterpreterContext context) {
+    synchronized (this) {
+      InterpreterResult r = interpretInput(lines);
+      return r;
+    }
+  }
+
+  public InterpreterResult interpretInput(String[] lines) {
+
+    // add print("") to make sure not finishing with comment
+    // see https://github.com/NFLabs/zeppelin/issues/151
+    String[] linesToRun = new String[lines.length + 1];
+    for (int i = 0; i < lines.length; i++) {
+      linesToRun[i] = lines[i];
+    }
+    linesToRun[lines.length] = "print(\"\")";
+
+    Console.setOut((java.io.PrintStream) binder.get("out"));
+    out.reset();
+    Code r = null;
+    String incomplete = "";
+
+    for (int l = 0; l < linesToRun.length; l++) {
+      String s = linesToRun[l];
+      // check if next line starts with "." (but not ".." or "./") it is treated as an invocation
+      if (l + 1 < linesToRun.length) {
+        String nextLine = linesToRun[l + 1].trim();
+        if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
+          incomplete += s + "\n";
+          continue;
+        }
+      }
+
+      scala.tools.nsc.interpreter.Results.Result res = null;
+      try {
+        res = interpreter.intp().interpret(incomplete + s);
+      } catch (Exception e) {
+        logger.error("Interpreter exception: ", e);
+        return new InterpreterResult(Code.ERROR, e.getMessage());
+      }
+
+      r = getResultCode(res);
+
+      if (r == Code.ERROR) {
+        Console.flush();
+        return new InterpreterResult(r, out.toString());
+      } else if (r == Code.INCOMPLETE) {
+        incomplete += s + "\n";
+      } else {
+        incomplete = "";
+      }
+    }
+
+    if (r == Code.INCOMPLETE) {
+      return new InterpreterResult(r, "Incomplete expression");
+    } else {
+      Console.flush();
+      return new InterpreterResult(r, out.toString());
+    }
+  }
+
+  private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) {
+    if (r instanceof scala.tools.nsc.interpreter.Results.Success$) {
+      return Code.SUCCESS;
+    } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) {
+      return Code.INCOMPLETE;
+    } else {
+      return Code.ERROR;
+    }
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+    // not implemented
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.NATIVE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    // fine-grained progress not implemented - return 0
+    return 0;
+  }
+
+  @Override
+  public Scheduler getScheduler() {
+    return SchedulerFactory.singleton().createOrGetFIFOScheduler(
+        ScaldingInterpreter.class.getName() + this.hashCode());
+  }
+
+  @Override
+  public List<String> completion(String buf, int cursor) {
+    return NO_COMPLETION;
+  }
+}

scalding/src/main/scala/org/apache/zeppelin/scalding/ScaldingILoop.scala

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.scalding;
+
+import java.io.{BufferedReader, File, FileReader}
+
+import scala.tools.nsc.GenericRunnerSettings
+import scala.tools.nsc.interpreter.{ILoop, IR, JPrintWriter}
+
+
+/**
+ * A class providing Scalding specific commands for inclusion in the Scalding REPL.
+ * This is currently forked from Scalding, but should eventually make it into Scalding itself:
+ * https://github.com/twitter/scalding/blob/develop/scalding-repl/src/main/scala/com/twitter/scalding/ScaldingILoop.scala
+ */
+ class ScaldingILoop(in0: Option[BufferedReader], out: JPrintWriter)
+    extends ILoop(in0, out) {
+  // def this(in0: BufferedReader, out: JPrintWriter) = this(Some(in0), out)
+  // def this() = this(None, new JPrintWriter(Console.out, true))
+
+  settings = new GenericRunnerSettings({ s => echo(s) })
+
+  override def printWelcome() {
+    val fc = Console.YELLOW
+    val wc = Console.RED
+    def wrapFlames(s: String) = s.replaceAll("[()]+", fc + "$0" + wc)
+    echo(fc +
+      " (                                           \n" +
+      " )\\ )            (   (                       \n" +
+      "(()/(         )  )\\  )\\ )  (          (  (   \n" +
+      " /(_)) (   ( /( ((_)(()/( )\\   (     )\\))(  \n" +
+      "(_))   )\\  )( )) _   ((_)(( )  )\\ ) (( ))\\  \n".replaceAll("_", wc + "_" + fc) + wc +
+      wrapFlames("/ __|((_) ((_)_ | |  _| | (_) _(_(( (_()_) \n") +
+      wrapFlames("\\__ \\/ _| / _` || |/ _` | | || ' \\))/ _` \\  \n") +
+      "|___/\\__| \\__,_||_|\\__,_| |_||_||_| \\__, |  \n" +
+      "                                    |___/   ")
+  }
+
+  /**
+   * Commands specific to the Scalding REPL. To define a new command use one of the following
+   * factory methods:
+   * - `LoopCommand.nullary` for commands that take no arguments
+   * - `LoopCommand.cmd` for commands that take one string argument
+   * - `LoopCommand.varargs` for commands that take multiple string arguments
+   */
+  private val scaldingCommands: List[LoopCommand] = List()
+
+  /**
+   * Change the shell prompt to read scalding>
+   *
+   * @return a prompt string to use for this REPL.
+   */
+  override def prompt: String = Console.BLUE + "\nscalding> " + Console.RESET
+
+  private[this] def addImports(ids: String*): IR.Result =
+    if (ids.isEmpty) IR.Success
+    else intp.interpret("import " + ids.mkString(", "))
+
+  /**
+   * Search for files with the given name in all directories from current directory
+   * up to root.
+   */
+  private def findAllUpPath(filename: String): List[File] =
+    Iterator.iterate(System.getProperty("user.dir"))(new File(_).getParent)
+      .takeWhile(_ != "/")
+      .flatMap(new File(_).listFiles.filter(_.toString.endsWith(filename)))
+      .toList
+
+  /**
+   * Gets the list of commands that this REPL supports.
+   *
+   * @return a list of the command supported by this REPL.
+   */
+  override def commands: List[LoopCommand] = super.commands ++ scaldingCommands
+
+  protected def imports: List[String] = List(
+    "com.twitter.scalding._",
+    "com.twitter.scalding.ReplImplicits._",
+    "com.twitter.scalding.ReplImplicitContext._",
+    "com.twitter.scalding.ReplState._")
+
+  override def createInterpreter() {
+    super.createInterpreter()
+    intp.beQuietDuring {
+      addImports(imports: _*)
+
+      settings match {
+        case s: GenericRunnerSettings =>
+          findAllUpPath(".scalding_repl").reverse.foreach {
+            f => s.loadfiles.appendToValue(f.toString)
+          }
+        case _ => ()
+      }
+    }
+  }
+}

scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.scalding;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Properties;
+
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+/**
+ * Tests for the Scalding interpreter for Zeppelin.
+ *
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class ScaldingInterpreterTest {
+  public static ScaldingInterpreter repl;
+  private InterpreterContext context;
+  private File tmpDir;
+
+  @Before
+  public void setUp() throws Exception {
+    tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
+    System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
+
+    tmpDir.mkdirs();
+
+    if (repl == null) {
+      Properties p = new Properties();
+
+      repl = new ScaldingInterpreter(p);
+      repl.open();
+    }
+
+    InterpreterGroup intpGroup = new InterpreterGroup();
+    context = new InterpreterContext("note", "id", "title", "text",
+        new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
+            intpGroup.getId(), null),
+        new LinkedList<InterpreterContextRunner>());
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    delete(tmpDir);
+    repl.close();
+  }
+
+  private void delete(File file) {
+    if (file.isFile()) file.delete();
+    else if (file.isDirectory()) {
+      File[] files = file.listFiles();
+      if (files != null && files.length > 0) {
+        for (File f : files) {
+          delete(f);
+        }
+      }
+      file.delete();
+    }
+  }
+
+  @Test
+  public void testBasicIntp() {
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        repl.interpret("val a = 1\nval b = 2", context).code());
+
+    // when interpret incomplete expression
+    InterpreterResult incomplete = repl.interpret("val a = \"\"\"", context);
+    assertEquals(InterpreterResult.Code.INCOMPLETE, incomplete.code());
+    assertTrue(incomplete.message().length() > 0); // expecting some error
+                                                   // message
+  }
+
+  @Test
+  public void testBasicScalding() {
+    assertEquals(InterpreterResult.Code.SUCCESS,
+        repl.interpret("case class Sale(state: String, name: String, sale: Int)\n" +
+          "val salesList = List(Sale(\"CA\", \"A\", 60), Sale(\"CA\", \"A\", 20), Sale(\"VA\", \"B\", 15))\n" +
+          "val salesPipe = TypedPipe.from(salesList)\n" +
+          "val results = salesPipe.map{x => (1, Set(x.state), x.sale)}.\n" +
+          "    groupAll.sum.values.map{ case(count, set, sum) => (count, set.size, sum) }\n" +
+          "results.dump", 
+          context).code());
+  }
+
+  @Test
+  public void testNextLineInvocation() {
+    assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n.toInt", context).code());
+  }
+
+  @Test
+  public void testEndWithComment() {
+    assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code());
+  }
+
+  @Test
+  public void testReferencingUndefinedVal() {
+    InterpreterResult result = repl.interpret("def category(min: Int) = {"
+        + "    if (0 <= value) \"error\"" + "}", context);
+    assertEquals(Code.ERROR, result.code());
+  }
+}

scripts/vagrant/zeppelin-dev/Vagrantfile

@@ -25,7 +25,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # please see the online documentation at vagrantup.com.
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "hashicorp/precise64"
+  # ubuntu/trusty64 Official Ubuntu Server 14.04 LTS (Trusty Tahr) builds
+  config.vm.box = "ubuntu/trusty64"
 
   # Disable automatic box update checking. If you disable this, then
   # boxes will only be checked for updates when the user runs

scripts/vagrant/zeppelin-dev/show-instructions.sh

@@ -36,4 +36,6 @@ echo '# or for a specific build'
 echo
 echo 'mvn clean package -Pspark-1.5 -Ppyspark -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests'
 echo './bin/zeppelin-daemon.sh start'
+echo
+echo 'On your host machine browse to http://localhost:8080/'
 

shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java

@@ -38,10 +38,6 @@ import org.slf4j.LoggerFactory;
 
 /**
  * Shell interpreter for Zeppelin.
- *
- * @author Leemoonsoo
- * @author anthonycorbacho
- *
  */
 public class ShellInterpreter extends Interpreter {
   Logger logger = LoggerFactory.getLogger(ShellInterpreter.class);

spark-dependencies/pom.xml

@@ -33,7 +33,7 @@
   <name>Zeppelin: Spark dependencies</name>
   <description>Zeppelin spark support</description>
   <url>http://zeppelin.incubator.apache.org</url>
-  
+
 
   <properties>
     <spark.version>1.4.1</spark.version>
@@ -130,117 +130,117 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>		
-        <groupId>org.apache.hadoop</groupId>		
-        <artifactId>hadoop-yarn-api</artifactId>		
-        <version>${yarn.version}</version>		
-        <exclusions>		
-          <exclusion>		
-            <groupId>asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.ow2.asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.jboss.netty</groupId>		
-            <artifactId>netty</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>commons-logging</groupId>		
-            <artifactId>commons-logging</artifactId>		
-          </exclusion>		
-        </exclusions>		
-      </dependency>		
-		
-      <dependency>		
-        <groupId>org.apache.hadoop</groupId>		
-        <artifactId>hadoop-yarn-common</artifactId>		
-        <version>${yarn.version}</version>		
-        <exclusions>		
-          <exclusion>		
-            <groupId>asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.ow2.asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.jboss.netty</groupId>		
-            <artifactId>netty</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>javax.servlet</groupId>		
-            <artifactId>servlet-api</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>commons-logging</groupId>		
-            <artifactId>commons-logging</artifactId>		
-          </exclusion>		
-        </exclusions>		
-      </dependency>		
-		
-      <dependency>		
-        <groupId>org.apache.hadoop</groupId>		
-        <artifactId>hadoop-yarn-server-web-proxy</artifactId>		
-        <version>${yarn.version}</version>		
-        <exclusions>		
-          <exclusion>		
-            <groupId>asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.ow2.asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.jboss.netty</groupId>		
-            <artifactId>netty</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>javax.servlet</groupId>		
-            <artifactId>servlet-api</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>commons-logging</groupId>		
-            <artifactId>commons-logging</artifactId>		
-          </exclusion>		
-        </exclusions>		
-      </dependency>		
-		
-      <dependency>		
-        <groupId>org.apache.hadoop</groupId>		
-        <artifactId>hadoop-yarn-client</artifactId>		
-        <version>${yarn.version}</version>		
-        <exclusions>		
-          <exclusion>		
-            <groupId>asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.ow2.asm</groupId>		
-            <artifactId>asm</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>org.jboss.netty</groupId>		
-            <artifactId>netty</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>javax.servlet</groupId>		
-            <artifactId>servlet-api</artifactId>		
-          </exclusion>		
-          <exclusion>		
-            <groupId>commons-logging</groupId>		
-            <artifactId>commons-logging</artifactId>		
-          </exclusion>		
-        </exclusions>		
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-api</artifactId>
+        <version>${yarn.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-common</artifactId>
+        <version>${yarn.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+        <version>${yarn.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-client</artifactId>
+        <version>${yarn.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.ow2.asm</groupId>
+            <artifactId>asm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jboss.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
     </dependencies>
   </dependencyManagement>
-  
-  <dependencies>  
+
+  <dependencies>
     <!-- Spark -->
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -489,7 +489,7 @@
       <dependencies>
       </dependencies>
     </profile>
- 
+
     <profile>
       <id>cassandra-spark-1.5</id>
       <properties>
@@ -513,7 +513,21 @@
         </dependency>
       </dependencies>
     </profile>
-   
+
+    <profile>
+      <id>spark-1.6</id>
+      <properties>
+        <spark.version>1.6.0</spark.version>
+        <py4j.version>0.9</py4j.version>
+        <akka.group>com.typesafe.akka</akka.group>
+        <akka.version>2.3.11</akka.version>
+        <protobuf.version>2.5.0</protobuf.version>
+      </properties>
+
+      <dependencies>
+      </dependencies>
+    </profile>
+
     <profile>
       <id>hadoop-0.23</id>
       <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a
@@ -731,10 +745,6 @@
 
     <profile>
       <id>pyspark</id>
-      <properties>
-        <spark.download.url>http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz
-        </spark.download.url>
-      </properties>
       <build>
         <plugins>
           <plugin>

spark/pom.xml

@@ -33,16 +33,70 @@
   <name>Zeppelin: Spark</name>
   <description>Zeppelin spark support</description>
   <url>http://zeppelin.incubator.apache.org</url>
-  
+
   <properties>
     <spark.version>1.4.1</spark.version>
     <scala.version>2.10.4</scala.version>
     <scala.binary.version>2.10</scala.binary.version>
-
     <hadoop.version>2.3.0</hadoop.version>
     <py4j.version>0.8.2.1</py4j.version>
   </properties>
 
+  <profiles>
+    <profile>
+      <id>vendor-repo</id>
+      <repositories>
+        <repository>
+          <id>cloudera</id>
+          <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+        </repository>
+      </repositories>
+    </profile>
+
+    <profile>
+      <id>spark-1.1</id>
+      <properties>
+        <spark.version>1.1.1</spark.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>spark-1.2</id>
+      <properties>
+        <spark.version>1.2.1</spark.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>spark-1.3</id>
+      <properties>
+        <spark.version>1.3.1</spark.version>
+      </properties>
+    </profile>
+
+   <profile>
+      <id>spark-1.4</id>
+      <properties>
+        <spark.version>1.4.1</spark.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>spark-1.5</id>
+      <properties>
+        <spark.version>1.5.2</spark.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>spark-1.6</id>
+      <properties>
+        <spark.version>1.6.0</spark.version>
+        <py4j.version>0.9</py4j.version>
+      </properties>
+    </profile>
+  </profiles>
+
   <dependencies>
     <dependency>
       <groupId>org.slf4j</groupId>
@@ -72,7 +126,7 @@
       <artifactId>guava</artifactId>
       <version>14.0.1</version>
     </dependency>
-    
+
     <!-- Aether :: maven dependency resolution -->
     <dependency>
       <groupId>org.apache.maven</groupId>
@@ -294,18 +348,6 @@
     </dependency>
   </dependencies>
 
-  <profiles>
-    <profile>
-      <id>vendor-repo</id>
-      <repositories>
-        <repository>
-          <id>cloudera</id>
-          <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
-        </repository>
-      </repositories>
-    </profile>
-  </profiles>
-
   <build>
     <plugins>
       <plugin>
@@ -397,7 +439,7 @@
               <overWriteReleases>false</overWriteReleases>
               <overWriteSnapshots>false</overWriteSnapshots>
               <overWriteIfNewer>true</overWriteIfNewer>
-              <includeScope>runtime</includeScope>              
+              <includeScope>runtime</includeScope>
               <artifactItems>
                 <artifactItem>
                   <groupId>${project.groupId}</groupId>

spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java

@@ -41,6 +41,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
 import org.apache.zeppelin.scheduler.Scheduler;
 import org.apache.zeppelin.spark.dep.DependencyContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonatype.aether.resolution.ArtifactResolutionException;
 import org.sonatype.aether.resolution.DependencyResolutionException;
 
@@ -80,6 +82,7 @@ public class DepInterpreter extends Interpreter {
   private DependencyContext depc;
   private SparkJLineCompletion completor;
   private SparkILoop interpreter;
+  static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class);
 
   public DepInterpreter(Properties property) {
     super(property);
@@ -195,6 +198,7 @@ public class DepInterpreter extends Interpreter {
       depc.fetch();
     } catch (MalformedURLException | DependencyResolutionException
         | ArtifactResolutionException e) {
+      LOGGER.error("Exception in DepInterpreter while interpret ", e);
       return new InterpreterResult(Code.ERROR, e.toString());
     }
 

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -82,7 +82,7 @@ import scala.tools.nsc.settings.MutableSettings.PathSetting;
  *
  */
 public class SparkInterpreter extends Interpreter {
-  Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
+  public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
 
   static {
     Interpreter.register(
@@ -186,7 +186,7 @@ public class SparkInterpreter extends Interpreter {
       }
     } catch (NoSuchMethodException | SecurityException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException e) {
-      e.printStackTrace();
+      logger.error(e.toString(), e);
       return null;
     }
     return pl;
@@ -317,7 +317,8 @@ public class SparkInterpreter extends Interpreter {
           "python" + File.separator + "lib");
     }
 
-    String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
+    //Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
+    String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip"};
     ArrayList<String> pythonLibUris = new ArrayList<>();
     for (String lib : pythonLibs) {
       File libFile = new File(pysparkPath, lib);
@@ -334,6 +335,10 @@ public class SparkInterpreter extends Interpreter {
       conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
     }
 
+    // Distributes needed libraries to workers.
+    if (getProperty("master").equals("yarn-client")) {
+      conf.set("spark.yarn.isPython", "true");
+    }
 
     SparkContext sparkContext = new SparkContext(conf);
     return sparkContext;

spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java

@@ -41,9 +41,6 @@ import org.slf4j.LoggerFactory;
 
 /**
  * Spark SQL interpreter for Zeppelin.
- *
- * @author Leemoonsoo
- *
  */
 public class SparkSqlInterpreter extends Interpreter {
   Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class);

spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java

@@ -32,9 +32,10 @@ public class SparkVersion {
   public static final SparkVersion SPARK_1_4_0 = SparkVersion.fromVersionString("1.4.0");
   public static final SparkVersion SPARK_1_5_0 = SparkVersion.fromVersionString("1.5.0");
   public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0");
+  public static final SparkVersion SPARK_1_7_0 = SparkVersion.fromVersionString("1.7.0");
 
   public static final SparkVersion MIN_SUPPORTED_VERSION =  SPARK_1_0_0;
-  public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_6_0;
+  public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_7_0;
 
   private int version;
   private String versionString;

spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java

@@ -51,9 +51,6 @@ import scala.collection.Iterable;
 
 /**
  * Spark context for zeppelin.
- *
- * @author Leemoonsoo
- *
  */
 public class ZeppelinContext extends HashMap<String, Object> {
   private DependencyResolver dep;
@@ -298,30 +295,30 @@ public class ZeppelinContext extends HashMap<String, Object> {
     try {
       take = df.getClass().getMethod("take", int.class);
       rows = (Object[]) take.invoke(df, maxResult + 1);
-
     } catch (NoSuchMethodException | SecurityException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException | ClassCastException e) {
       sc.clearJobGroup();
       throw new InterpreterException(e);
     }
 
-    String msg = null;
-
+    List<Attribute> columns = null;
     // get field names
-    Method queryExecution;
-    QueryExecution qe;
     try {
-      queryExecution = df.getClass().getMethod("queryExecution");
-      qe = (QueryExecution) queryExecution.invoke(df);
+      // Use reflection because of classname returned by queryExecution changes from
+      // Spark <1.5.2 org.apache.spark.sql.SQLContext$QueryExecution
+      // Spark 1.6.0> org.apache.spark.sql.hive.HiveContext$QueryExecution
+      Object qe = df.getClass().getMethod("queryExecution").invoke(df);
+      Object a = qe.getClass().getMethod("analyzed").invoke(qe);
+      scala.collection.Seq seq = (scala.collection.Seq) a.getClass().getMethod("output").invoke(a);
+
+      columns = (List<Attribute>) scala.collection.JavaConverters.seqAsJavaListConverter(seq)
+                                                                 .asJava();
     } catch (NoSuchMethodException | SecurityException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException e) {
       throw new InterpreterException(e);
     }
 
-    List<Attribute> columns =
-        scala.collection.JavaConverters.asJavaListConverter(
-            qe.analyzed().output()).asJava();
-
+    String msg = null;
     for (Attribute col : columns) {
       if (msg == null) {
         msg = col.name();

spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java

@@ -27,9 +27,6 @@ import org.sonatype.aether.repository.RemoteRepository;
 
 /**
  * Manage mvn repository.
- *
- * @author anthonycorbacho
- *
  */
 public class Booter {
   public static RepositorySystem newRepositorySystem() {

spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java

@@ -57,9 +57,6 @@ import scala.tools.nsc.util.MergedClassPath;
 /**
  * Deps resolver.
  * Add new dependencies from mvn repo (at runetime) to Zeppelin.
- *
- * @author anthonycorbacho
- *
  */
 public class DependencyResolver {
   Logger logger = LoggerFactory.getLogger(DependencyResolver.class);

spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java

@@ -24,9 +24,6 @@ import org.sonatype.aether.RepositoryEvent;
 
 /**
  * Simple listener that print log.
- *
- * @author anthonycorbacho
- *
  */
 public class RepositoryListener extends AbstractRepositoryListener {
   Logger logger = LoggerFactory.getLogger(RepositoryListener.class);

spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java

@@ -29,9 +29,6 @@ import org.sonatype.aether.spi.connector.RepositoryConnectorFactory;
 
 /**
  * Get maven repository instance.
- *
- * @author anthonycorbacho
- *
  */
 public class RepositorySystemFactory {
   public static RepositorySystem newRepositorySystem() {

spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java

@@ -32,9 +32,6 @@ import org.sonatype.aether.transfer.TransferResource;
 
 /**
  * Simple listener that show deps downloading progress.
- *
- * @author anthonycorbacho
- *
  */
 public class TransferListener extends AbstractTransferListener {
   Logger logger = LoggerFactory.getLogger(TransferListener.class);

spark/src/main/resources/python/zeppelin_pyspark.py

@@ -111,7 +111,7 @@ class PySparkCompletion:
   def getGlobalCompletion(self):
     objectDefList = []
     try:
-      for completionItem in list(globals().iterkeys()):
+      for completionItem in list(globals().keys()):
         objectDefList.append(completionItem)
     except:
       return None
@@ -119,18 +119,20 @@ class PySparkCompletion:
       return objectDefList
 
   def getMethodCompletion(self, text_value):
-    objectDefList = []
+    execResult = locals()
+    if text_value == None:
+      return None
     completion_target = text_value
     try:
       if len(completion_target) <= 0:
         return None
       if text_value[-1] == ".":
         completion_target = text_value[:-1]
-      exec("%s = %s(%s)" % ("objectDefList", "dir", completion_target))
+      exec("{} = dir({})".format("objectDefList", completion_target), globals(), execResult)
     except:
       return None
     else:
-      return objectDefList
+      return list(execResult['objectDefList'])
 
 
   def getCompletion(self, text_value):
@@ -147,9 +149,9 @@ class PySparkCompletion:
         for completionItem in list(objectCompletionList):
           completionList.add(completionItem)
     if len(completionList) <= 0:
-      print ""
+      print("")
     else:
-      print json.dumps(filter(lambda x : not re.match("^__.*", x), list(completionList)))
+      print(json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList)))))
 
 
 output = Logger()

spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java

@@ -38,12 +38,15 @@ import org.junit.Before;
 import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class SparkInterpreterTest {
   public static SparkInterpreter repl;
   private InterpreterContext context;
   private File tmpDir;
+  public static Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterTest.class);
 
 
   /**
@@ -177,7 +180,7 @@ public class SparkInterpreterTest {
     for (Object oKey : intpProperty.keySet()) {
       String key = (String) oKey;
       String value = (String) intpProperty.get(key);
-      repl.logger.debug(String.format("[%s]: [%s]", key, value));
+      LOGGER.debug(String.format("[%s]: [%s]", key, value));
       if (key.startsWith("spark.") && value.isEmpty()) {
         assertTrue(String.format("configuration starting from 'spark.' should not be empty. [%s]", key), !sparkConf.contains(key) || !sparkConf.get(key).isEmpty());
       }

spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java

@@ -33,6 +33,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Type;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SparkSqlInterpreterTest {
 
@@ -41,6 +43,8 @@ public class SparkSqlInterpreterTest {
   private InterpreterContext context;
   private InterpreterGroup intpGroup;
 
+  Logger LOGGER = LoggerFactory.getLogger(SparkSqlInterpreterTest.class);
+
   @Before
   public void setUp() throws Exception {
     Properties p = new Properties();
@@ -96,6 +100,7 @@ public class SparkSqlInterpreterTest {
       fail("Exception not catched");
     } catch (Exception e) {
       // okay
+      LOGGER.info("Exception in SparkSqlInterpreterTest while test ", e);
     }
     assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from test", context).code());
   }

tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSet.java

@@ -17,13 +17,16 @@
  */
 package org.apache.zeppelin.tajo;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
 import java.math.BigDecimal;
 import java.sql.*;
 import java.util.Calendar;
 import java.util.Map;
-import java.io.InputStream;
-import java.io.Reader;
-import java.io.UnsupportedEncodingException;
 
 /**
  * This is borrowed from Apache Commons DBCP2.
@@ -31,6 +34,9 @@ import java.io.UnsupportedEncodingException;
  * A dummy {@link java.sql.ResultSet}, for testing purposes.
  */
 public class TesterResultSet implements ResultSet {
+
+  Logger LOGGER = LoggerFactory.getLogger(TesterResultSet.class);
+
   public TesterResultSet(Statement stmt) {
     _statement = stmt;
   }
@@ -262,6 +268,8 @@ public class TesterResultSet implements ResultSet {
       return columnName.getBytes("UTF-8");
     } catch (UnsupportedEncodingException e) {
       // Impossible. JVMs are required to support UTF-8
+      LOGGER.error("Exception in TesterResultSet while getBytes, Impossible. JVMs are required to" +
+          " support UTF-8", e);
       return null;
     }
   }

zeppelin-distribution/src/bin_license/LICENSE

@@ -1,4 +1,4 @@
-    (Apache 2.0) nvd3.js v1.1.15-beta (http://nvd3.org/) - https://github.com/novus/nvd3/blob/v1.1.15-beta/LICENSE.md
+    (Apache 2.0) nvd3.js v1.7.1 (http://nvd3.org/) - https://github.com/novus/nvd3/blob/v1.7.1/LICENSE.md
     (Apache 2.0) gson v2.2 (com.google.code.gson:gson:jar:2.2 - https://github.com/google/gson) - https://github.com/google/gson/blob/gson-2.2/LICENSE
     (Apache 2.0) Amazon Web Services SDK for Java v1.10.1 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.10.1/LICENSE.txt
     (Apache 2.0) JavaEWAH v0.7.9 (https://github.com/lemire/javaewah) - https://github.com/lemire/javaewah/blob/master/LICENSE-2.0.txt
@@ -10,11 +10,11 @@ The following components are provided under Apache License.
     (Apache 2.0) Apache Commons Logging (commons-logging:commons-logging:1.1.1 - http://commons.apache.org/proper/commons-logging/)
     (Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.5 - http://commons.apache.org/proper/commons-codec/)
     (Apache 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.1 - http://commons.apache.org/proper/commons-configuration/)
-    (Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.9 - http://commons.apache.org/proper/commons-compress/)    
+    (Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.9 - http://commons.apache.org/proper/commons-compress/)
     (Apache 2.0) Apache Commons Configuration (commons-configuration:commons-configuration:1.9 - http://commons.apache.org/configuration/)
     (Apache 2.0) Apache Commons CLI (commons-cli:commons-cli:1.2 - http://commons.apache.org/cli/)
     (Apache 2.0) Apache Commons Exec (commons-exec:commons-exec:1.3 - http://commons.apache.org/exec/)
-    (Apache 2.0) Http Components (org.apache.httpcomponents:httpcore:4.3.3 - https://github.com/apache/httpclient)    
+    (Apache 2.0) Http Components (org.apache.httpcomponents:httpcore:4.3.3 - https://github.com/apache/httpclient)
     (Apache 2.0) Http Components (org.apache.httpcomponents:httpclient:4.3.6 - https://github.com/apache/httpclient)
     (Apache 2.0) Apache Commons Lang (org.apache.commons:commons-lang:2.5 - http://commons.apache.org/proper/commons-lang/)
     (Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.4 - http://commons.apache.org/proper/commons-lang/)
@@ -54,7 +54,7 @@ The following components are provided under Apache License.
     (Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-annotations:2.5.0 - https://github.com/FasterXML/jackson-core)
     (Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-databind:2.5.3 - https://github.com/FasterXML/jackson-core)
     (Apache 2.0) javax.servlet (org.eclipse.jetty.orbit:javax.servlet:jar:3.0.0.v201112011016 - http://www.eclipse.org/jetty)
-    (Apache 2.0) Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/)    
+    (Apache 2.0) Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/)
     (Apache 2.0) Jackson (org.codehaus.jackson:jackson-core-asl:1.9.13 - http://jackson.codehaus.org/)
     (Apache 2.0) JetS3t (net.java.dev.jets3t:jets3t:jar:0.9.3) - http://www.jets3t.org/
     (Apache 2.0) Jetty (org.eclipse.jetty:jetty - http://www.eclipse.org/jetty)
@@ -91,6 +91,8 @@ The following components are provided under Apache License.
     (Apache 2.0) Lucene Suggest (org.apache.lucene:lucene-suggest:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-suggest)
     (Apache 2.0) Elasticsearch: Core (org.elasticsearch:elasticsearch:2.1.0 - http://nexus.sonatype.org/oss-repository-hosting.html/parent/elasticsearch)
     (Apache 2.0) Joda convert (org.joda:joda-convert:1.2 - http://joda-convert.sourceforge.net)
+    (Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org)
+    (Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org)
     (Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org)
 
 
@@ -111,6 +113,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (The MIT License) Angular Websocket v1.0.13 (http://angularclass.github.io/angular-websocket/) - https://github.com/AngularClass/angular-websocket/blob/v1.0.13/LICENSE
     (The MIT License) UI.Ace v0.1.1 (http://angularclass.github.io/angular-websocket/) - https://github.com/angular-ui/ui-ace/blob/master/LICENSE
     (The MIT License) jquery.scrollTo v1.4.13 (https://github.com/flesler/jquery.scrollTo) - https://github.com/flesler/jquery.scrollTo/blob/1.4.13/LICENSE
+    (The MIT License) jquery.floatThead v1.3.2 (https://github.com/mkoryak/floatThead) - https://github.com/mkoryak/floatThead/blob/master/license.txt
     (The MIT License) angular-dragdrop v1.0.8 (http://codef0rmer.github.io/angular-dragdrop/#/) - https://github.com/codef0rmer/angular-dragdrop/blob/v1.0.8/LICENSE
     (The MIT License) perfect-scrollbar v0.5.4 (http://noraesae.github.io/perfect-scrollbar/) - https://github.com/noraesae/perfect-scrollbar/tree/0.5.4
     (The MIT License) ng-sortable v1.1.9 (https://github.com/a5hik/ng-sortable) - https://github.com/a5hik/ng-sortable/blob/1.1.9/LICENSE
@@ -128,7 +131,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
 The following components are provided under the MIT License.
 
     (The MIT License) Objenesis (org.objenesis:objenesis:2.1 - https://github.com/easymock/objenesis) - Copyright (c) 2006-2015 the original author and authors
-    (The MIT License) JCL 1.1.1 implemented over SLF4J (org.slf4j:jcl-over-slf4j:1.7.5 - http://www.slf4j.org)    
+    (The MIT License) JCL 1.1.1 implemented over SLF4J (org.slf4j:jcl-over-slf4j:1.7.5 - http://www.slf4j.org)
     (The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.5 - http://www.slf4j.org)
     (The MIT License) angular-resource (angular-resource - https://github.com/angular/angular.js/tree/master/src/ngResource)
     (The MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.4 - https://github.com/ralfstx/minimal-json)
@@ -155,7 +158,7 @@ The following components are provided under the BSD-style License.
     (New BSD License) JGit (org.eclipse.jgit:org.eclipse.jgit:jar:4.1.1.201511131810-r - https://eclipse.org/jgit/)
     (New BSD License) Kryo (com.esotericsoftware.kryo:kryo:2.21 - http://code.google.com/p/kryo/)
     (New BSD License) MinLog (com.esotericsoftware.minlog:minlog:1.2 - http://code.google.com/p/minlog/)
-    (New BSD License) ReflectASM (com.esotericsoftware.reflectasm:reflectasm:1.07 - http://code.google.com/p/reflectasm/)    
+    (New BSD License) ReflectASM (com.esotericsoftware.reflectasm:reflectasm:1.07 - http://code.google.com/p/reflectasm/)
     (BSD-like) Scala Library (org.scala-lang:scala-library:2.10.4 - http://www.scala-lang.org/)
     (BSD-like) Scalap (org.scala-lang:scalap:2.10.4 - http://www.scala-lang.org/)
     (BSD-like) (The BSD License) jline (org.scala-lang:jline:2.10.4 - http://www.scala-lang.org/)
@@ -165,7 +168,7 @@ The following components are provided under the BSD-style License.
     (BSD-like) ASM (asm:asm:jar:3.1 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
     (New BSD License) Py4J (net.sf.py4j:py4j:0.9 - http://py4j.sourceforge.net/)
     (New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
-     
+
 
 
 ========================================================================
@@ -211,4 +214,3 @@ Creative Commons CC0 (http://creativecommons.org/publicdomain/zero/1.0/)
 
     (CC0 1.0 Universal) JSR166e (com.twitter:jsr166e:1.1.0 - http://github.com/twitter/jsr166e)
     (Public Domain, per Creative Commons CC0) HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.6 - http://hdrhistogram.github.io/HdrHistogram/)
-     

zeppelin-distribution/src/bin_license/licenses/LICENSE-nvd3.js-1.7.1

@@ -1,4 +1,4 @@
-Copyright (c) 2011, 2012 Novus Partners, Inc.
+Copyright (c) 2011-2014 Novus Partners, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/GUI.java

@@ -26,9 +26,6 @@ import org.apache.zeppelin.display.Input.ParamOption;
 
 /**
  * Settings of a form.
- *
- * @author Leemoonsoo
- *
  */
 public class GUI implements Serializable {
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java

@@ -28,16 +28,10 @@ import java.util.regex.Pattern;
 
 /**
  * Input type.
- *
- * @author Leemoonsoo
- *
  */
 public class Input implements Serializable {
   /**
    * Parameters option.
-   *
-   * @author Leemoonsoo
-   *
    */
   public static class ParamOption {
     Object value;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java

@@ -131,7 +131,7 @@ public abstract class Interpreter {
 
 
 
-  static Logger logger = LoggerFactory.getLogger(Interpreter.class);
+  public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
   private InterpreterGroup interpreterGroup;
   private URL [] classloaderUrls;
   protected Properties property;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java

@@ -24,6 +24,7 @@ import java.util.Random;
 
 import org.apache.log4j.Logger;
 import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
 
 /**
  * InterpreterGroup is list of interpreters in the same group.
@@ -32,7 +33,10 @@ import org.apache.zeppelin.display.AngularObjectRegistry;
 public class InterpreterGroup extends LinkedList<Interpreter>{
   String id;
 
+  Logger LOGGER = Logger.getLogger(InterpreterGroup.class);
+
   AngularObjectRegistry angularObjectRegistry;
+  RemoteInterpreterProcess remoteInterpreterProcess;    // attached remote interpreter process
 
   public InterpreterGroup(String id) {
     this.id = id;
@@ -72,6 +76,14 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
     this.angularObjectRegistry = angularObjectRegistry;
   }
 
+  public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+    return remoteInterpreterProcess;
+  }
+
+  public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
+    this.remoteInterpreterProcess = remoteInterpreterProcess;
+  }
+
   public void close() {
     List<Thread> closeThreads = new LinkedList<Thread>();
 
@@ -90,8 +102,7 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
       try {
         t.join();
       } catch (InterruptedException e) {
-        Logger logger = Logger.getLogger(InterpreterGroup.class);
-        logger.error("Can't close interpreter", e);
+        LOGGER.error("Can't close interpreter", e);
       }
     }
   }
@@ -114,8 +125,14 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
       try {
         t.join();
       } catch (InterruptedException e) {
-        Logger logger = Logger.getLogger(InterpreterGroup.class);
-        logger.error("Can't close interpreter", e);
+        LOGGER.error("Can't close interpreter", e);
+      }
+    }
+
+    // make sure remote interpreter process terminates
+    if (remoteInterpreterProcess != null) {
+      while (remoteInterpreterProcess.referenceCount() > 0) {
+        remoteInterpreterProcess.dereference();
       }
     }
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java

@@ -19,6 +19,7 @@ package org.apache.zeppelin.interpreter.remote;
 
 import java.util.List;
 
+import org.apache.thrift.TException;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.AngularObjectRegistryListener;
@@ -77,15 +78,19 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
     }
 
     Client client = null;
+    boolean broken = false;
     try {
       client = remoteInterpreterProcess.getClient();
       client.angularObjectAdd(name, noteId, gson.toJson(o));
       return super.add(name, o, noteId, true);
+    } catch (TException e) {
+      broken = true;
+      logger.error("Error", e);
     } catch (Exception e) {
       logger.error("Error", e);
     } finally {
       if (client != null) {
-        remoteInterpreterProcess.releaseClient(client);
+        remoteInterpreterProcess.releaseClient(client, broken);
       }
     }
     return null;
@@ -96,7 +101,6 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
    * this method should be used instead of remove()
    * @param name
    * @param noteId
-   * @param emit
    * @return
    */
   public AngularObject removeAndNotifyRemoteProcess(String name, String noteId) {
@@ -106,15 +110,19 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
     }
 
     Client client = null;
+    boolean broken = false;
     try {
       client = remoteInterpreterProcess.getClient();
       client.angularObjectRemove(name, noteId);
       return super.remove(name, noteId);
+    } catch (TException e) {
+      broken = true;
+      logger.error("Error", e);
     } catch (Exception e) {
       logger.error("Error", e);
     } finally {
       if (client != null) {
-        remoteInterpreterProcess.releaseClient(client);
+        remoteInterpreterProcess.releaseClient(client, broken);
       }
     }
     return null;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java

@@ -56,8 +56,6 @@ public class RemoteInterpreter extends Interpreter {
   FormType formType;
   boolean initialized;
   private Map<String, String> env;
-  static Map<String, RemoteInterpreterProcess> interpreterGroupReference
-    = new HashMap<String, RemoteInterpreterProcess>();
 
   private int connectTimeout;
 
@@ -96,19 +94,21 @@ public class RemoteInterpreter extends Interpreter {
   }
 
   public RemoteInterpreterProcess getInterpreterProcess() {
-    synchronized (interpreterGroupReference) {
-      if (interpreterGroupReference.containsKey(getInterpreterGroupKey(getInterpreterGroup()))) {
-        RemoteInterpreterProcess interpreterProcess = interpreterGroupReference
-            .get(getInterpreterGroupKey(getInterpreterGroup()));
-        try {
-          return interpreterProcess;
-        } catch (Exception e) {
-          throw new InterpreterException(e);
-        }
-      } else {
-        // closed or not opened yet
-        return null;
+    InterpreterGroup intpGroup = getInterpreterGroup();
+    if (intpGroup == null) {
+      return null;
+    }
+
+    synchronized (intpGroup) {
+      if (intpGroup.getRemoteInterpreterProcess() == null) {
+        // create new remote process
+        RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess(
+                interpreterRunner, interpreterPath, env, connectTimeout);
+
+        intpGroup.setRemoteInterpreterProcess(remoteProcess);
       }
+
+      return intpGroup.getRemoteInterpreterProcess();
     }
   }
 
@@ -117,17 +117,7 @@ public class RemoteInterpreter extends Interpreter {
       return;
     }
 
-    RemoteInterpreterProcess interpreterProcess = null;
-
-    synchronized (interpreterGroupReference) {
-      if (interpreterGroupReference.containsKey(getInterpreterGroupKey(getInterpreterGroup()))) {
-        interpreterProcess = interpreterGroupReference
-            .get(getInterpreterGroupKey(getInterpreterGroup()));
-      } else {
-        throw new InterpreterException("Unexpected error");
-      }
-    }
-
+    RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
     int rc = interpreterProcess.reference(getInterpreterGroup());
 
     synchronized (interpreterProcess) {
@@ -141,6 +131,7 @@ public class RemoteInterpreter extends Interpreter {
           throw new InterpreterException(e1);
         }
 
+        boolean broken = false;
         try {
           for (Interpreter intp : this.getInterpreterGroup()) {
             logger.info("Create remote interpreter {}", intp.getClassName());
@@ -148,9 +139,10 @@ public class RemoteInterpreter extends Interpreter {
 
           }
         } catch (TException e) {
+          broken = true;
           throw new InterpreterException(e);
         } finally {
-          interpreterProcess.releaseClient(client);
+          interpreterProcess.releaseClient(client, broken);
         }
       }
     }
@@ -168,26 +160,21 @@ public class RemoteInterpreter extends Interpreter {
   public void close() {
     RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
     Client client = null;
+
+    boolean broken = false;
     try {
       client = interpreterProcess.getClient();
-    } catch (Exception e1) {
-      throw new InterpreterException(e1);
-    }
-
-    try {
       client.close(className);
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
+    } catch (Exception e1) {
+      throw new InterpreterException(e1);
     } finally {
-      interpreterProcess.releaseClient(client);
-    }
-
-    int r = interpreterProcess.dereference();
-    if (r == 0) {
-      synchronized (interpreterGroupReference) {
-        InterpreterGroup intpGroup = getInterpreterGroup();
-        interpreterGroupReference.remove(getInterpreterGroupKey(intpGroup));
+      if (client != null) {
+        interpreterProcess.releaseClient(client, broken);
       }
+      getInterpreterProcess().dereference();
     }
   }
 
@@ -215,6 +202,7 @@ public class RemoteInterpreter extends Interpreter {
       interpreterContextRunnerPool.addAll(noteId, runners);
     }
 
+    boolean broken = false;
     try {
       GUI settings = context.getGui();
       RemoteInterpreterResult remoteResult = client.interpret(className, st, convert(context));
@@ -235,9 +223,10 @@ public class RemoteInterpreter extends Interpreter {
       InterpreterResult result = convert(remoteResult);
       return result;
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
     } finally {
-      interpreterProcess.releaseClient(client);
+      interpreterProcess.releaseClient(client, broken);
     }
   }
 
@@ -251,12 +240,14 @@ public class RemoteInterpreter extends Interpreter {
       throw new InterpreterException(e1);
     }
 
+    boolean broken = false;
     try {
       client.cancel(className, convert(context));
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
     } finally {
-      interpreterProcess.releaseClient(client);
+      interpreterProcess.releaseClient(client, broken);
     }
   }
 
@@ -277,13 +268,15 @@ public class RemoteInterpreter extends Interpreter {
       throw new InterpreterException(e1);
     }
 
+    boolean broken = false;
     try {
       formType = FormType.valueOf(client.getFormType(className));
       return formType;
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
     } finally {
-      interpreterProcess.releaseClient(client);
+      interpreterProcess.releaseClient(client, broken);
     }
   }
 
@@ -297,12 +290,14 @@ public class RemoteInterpreter extends Interpreter {
       throw new InterpreterException(e1);
     }
 
+    boolean broken = false;
     try {
       return client.getProgress(className, convert(context));
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
     } finally {
-      interpreterProcess.releaseClient(client);
+      interpreterProcess.releaseClient(client, broken);
     }
   }
 
@@ -317,12 +312,14 @@ public class RemoteInterpreter extends Interpreter {
       throw new InterpreterException(e1);
     }
 
+    boolean broken = false;
     try {
       return client.completion(className, buf, cursor);
     } catch (TException e) {
+      broken = true;
       throw new InterpreterException(e);
     } finally {
-      interpreterProcess.releaseClient(client);
+      interpreterProcess.releaseClient(client, broken);
     }
   }
 
@@ -339,29 +336,6 @@ public class RemoteInterpreter extends Interpreter {
     }
   }
 
-
-  @Override
-  public void setInterpreterGroup(InterpreterGroup interpreterGroup) {
-    super.setInterpreterGroup(interpreterGroup);
-
-    synchronized (interpreterGroupReference) {
-      RemoteInterpreterProcess intpProcess = interpreterGroupReference
-          .get(getInterpreterGroupKey(interpreterGroup));
-
-      // when interpreter process is not created or terminated
-      if (intpProcess == null || (!intpProcess.isRunning() && intpProcess.getPort() > 0)
-          || (!intpProcess.isRunning() && intpProcess.getPort() == -1)) {
-        interpreterGroupReference.put(getInterpreterGroupKey(interpreterGroup),
-            new RemoteInterpreterProcess(interpreterRunner,
-                interpreterPath, env, connectTimeout));
-
-        logger.info("setInterpreterGroup = "
-            + getInterpreterGroupKey(interpreterGroup) + " class=" + className
-            + ", path=" + interpreterPath);
-      }
-    }
-  }
-
   private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) {
     return interpreterGroup.getId();
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java

@@ -66,16 +66,18 @@ public class RemoteInterpreterEventPoller extends Thread {
       }
 
       RemoteInterpreterEvent event = null;
+      boolean broken = false;
       try {
         event = client.getEvent();
       } catch (TException e) {
+        broken = true;
         logger.error("Can't get RemoteInterpreterEvent", e);
         waitQuietly();
         continue;
+      } finally {
+        interpreterProcess.releaseClient(client, broken);
       }
 
-      interpreterProcess.releaseClient(client);
-
       Gson gson = new Gson();
 
       AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
@@ -122,6 +124,7 @@ public class RemoteInterpreterEventPoller extends Thread {
         wait(1000);
       }
     } catch (InterruptedException ignored) {
+      logger.info("Error in RemoteInterpreterEventPoller while waitQuietly : ", ignored);
     }
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java

@@ -121,6 +121,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
             try {
               Thread.sleep(500);
             } catch (InterruptedException e) {
+              logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
+                  "Thread.sleep", e);
             }
           }
         }
@@ -140,7 +142,27 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
   }
 
   public void releaseClient(Client client) {
-    clientPool.returnObject(client);
+    releaseClient(client, false);
+  }
+
+  public void releaseClient(Client client, boolean broken) {
+    if (broken) {
+      releaseBrokenClient(client);
+    } else {
+      try {
+        clientPool.returnObject(client);
+      } catch (Exception e) {
+        logger.warn("exception occurred during releasing thrift client", e);
+      }
+    }
+  }
+
+  public void releaseBrokenClient(Client client) {
+    try {
+      clientPool.invalidateObject(client);
+    } catch (Exception e) {
+      logger.warn("exception occurred during releasing thrift client", e);
+    }
   }
 
   public int dereference() {
@@ -157,9 +179,12 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
           client.shutdown();
         } catch (Exception e) {
           // safely ignore exception while client.shutdown() may terminates remote process
+          logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
+              "safely ignore exception while client.shutdown() may terminates remote process", e);
         } finally {
           if (client != null) {
-            releaseClient(client);
+            // no longer used
+            releaseBrokenClient(client);
           }
         }
 
@@ -174,6 +199,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
             try {
               Thread.sleep(500);
             } catch (InterruptedException e) {
+              logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
+                  "Thread.sleep", e);
             }
           } else {
             break;
@@ -245,18 +272,22 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
       client = getClient();
     } catch (NullPointerException e) {
       // remote process not started
+      logger.info("NullPointerException in RemoteInterpreterProcess while " +
+          "updateRemoteAngularObject getClient, remote process not started", e);
       return;
     } catch (Exception e) {
       logger.error("Can't update angular object", e);
     }
 
+    boolean broken = false;
     try {
       Gson gson = new Gson();
       client.angularObjectUpdate(name, noteId, gson.toJson(o));
     } catch (TException e) {
+      broken = true;
       logger.error("Can't update angular object", e);
     } finally {
-      releaseClient(client);
+      releaseClient(client, broken);
     }
   }
 

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -114,6 +114,7 @@ public class RemoteInterpreterServer
       try {
         Thread.sleep(300);
       } catch (InterruptedException e) {
+        logger.info("Exception in RemoteInterpreterServer while shutdown, Thread.sleep", e);
       }
     }
 
@@ -170,7 +171,7 @@ public class RemoteInterpreterServer
     } catch (ClassNotFoundException | NoSuchMethodException | SecurityException
         | InstantiationException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException e) {
-      e.printStackTrace();
+      logger.error(e.toString(), e);
       throw new TException(e);
     }
   }
@@ -225,6 +226,7 @@ public class RemoteInterpreterServer
         try {
           jobListener.wait(1000);
         } catch (InterruptedException e) {
+          logger.info("Exception in RemoteInterpreterServer while interpret, jobListener.wait", e);
         }
       }
     }
@@ -455,6 +457,7 @@ public class RemoteInterpreterServer
         try {
           eventQueue.wait(1000);
         } catch (InterruptedException e) {
+          logger.info("Exception in RemoteInterpreterServer while getEvent, eventQueue.wait", e);
         }
       }
 
@@ -468,7 +471,6 @@ public class RemoteInterpreterServer
 
   /**
    * called when object is updated in client (web) side.
-   * @param className
    * @param name
    * @param noteId noteId where the update issues
    * @param object
@@ -499,6 +501,7 @@ public class RemoteInterpreterServer
         return;
       } catch (Exception e) {
         // no luck
+        logger.info("Exception in RemoteInterpreterServer while angularObjectUpdate, no luck", e);
       }
     }
 
@@ -510,6 +513,7 @@ public class RemoteInterpreterServer
           }.getType());
       } catch (Exception e) {
         // no lock
+        logger.info("Exception in RemoteInterpreterServer while angularObjectUpdate, no lock", e);
       }
     }
 
@@ -544,6 +548,7 @@ public class RemoteInterpreterServer
           }.getType());
     } catch (Exception e) {
       // nolock
+      logger.info("Exception in RemoteInterpreterServer while angularObjectAdd, nolock", e);
     }
 
     // try string object type at last

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java

@@ -17,6 +17,9 @@
 
 package org.apache.zeppelin.interpreter.remote;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
@@ -26,6 +29,7 @@ import java.net.Socket;
  *
  */
 public class RemoteInterpreterUtils {
+  static Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreterUtils.class);
   public static int findRandomAvailablePortOnAllLocalInterfaces() throws IOException {
     int port;
     try (ServerSocket socket = new ServerSocket(0);) {
@@ -43,6 +47,7 @@ public class RemoteInterpreterUtils {
       discover.close();
       return true;
     } catch (IOException e) {
+      LOGGER.info("Exception in RemoteInterpreterUtils while checkIfRemoteEndpointAccessible", e);
       return false;
     }
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/FIFOScheduler.java

@@ -24,6 +24,8 @@ import java.util.List;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.zeppelin.scheduler.Job.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * FIFOScheduler runs submitted job sequentially
@@ -36,6 +38,8 @@ public class FIFOScheduler implements Scheduler {
   Job runningJob = null;
   private String name;
 
+  static Logger LOGGER = LoggerFactory.getLogger(FIFOScheduler.class);
+
   public FIFOScheduler(String name, ExecutorService executor, SchedulerListener listener) {
     this.name = name;
     this.executor = executor;
@@ -107,6 +111,7 @@ public class FIFOScheduler implements Scheduler {
             try {
               queue.wait(500);
             } catch (InterruptedException e) {
+              LOGGER.error("Exception in FIFOScheduler while run queue.wait", e);
             }
             continue;
           }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java

@@ -57,15 +57,15 @@ public abstract class Job {
     FINISHED,
     ERROR,
     ABORT;
-    boolean isReady() {
+    public boolean isReady() {
       return this == READY;
     }
 
-    boolean isRunning() {
+    public boolean isRunning() {
       return this == RUNNING;
     }
 
-    boolean isPending() {
+    public boolean isPending() {
       return this == PENDING;
     }
   }
@@ -78,6 +78,8 @@ public abstract class Job {
   Date dateFinished;
   Status status;
 
+  static Logger LOGGER = LoggerFactory.getLogger(Job.class);
+
   transient boolean aborted = false;
 
   String errorMessage;
@@ -172,14 +174,14 @@ public abstract class Job {
       dateFinished = new Date();
       progressUpdator.terminate();
     } catch (NullPointerException e) {
-      logger().error("Job failed", e);
+      LOGGER.error("Job failed", e);
       progressUpdator.terminate();
       this.exception = e;
       result = e.getMessage();
       errorMessage = getStack(e);
       dateFinished = new Date();
     } catch (Throwable e) {
-      logger().error("Job failed", e);
+      LOGGER.error("Job failed", e);
       progressUpdator.terminate();
       this.exception = e;
       result = e.getMessage();
@@ -248,10 +250,6 @@ public abstract class Job {
     return dateFinished;
   }
 
-  private Logger logger() {
-    return LoggerFactory.getLogger(Job.class);
-  }
-
   protected void setResult(Object result) {
     this.result = result;
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobListener.java

@@ -19,9 +19,6 @@ package org.apache.zeppelin.scheduler;
 
 /**
  * TODO(moon) : add description.
- *
- * @author Leemoonsoo
- *
  */
 public interface JobListener {
   public void onProgressUpdate(Job job, int progress);

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java

@@ -22,9 +22,6 @@ import org.slf4j.LoggerFactory;
 
 /**
  * TODO(moon) : add description.
- *
- * @author Leemoonsoo
- *
  */
 public class JobProgressPoller extends Thread {
   public static final long DEFAULT_INTERVAL_MSEC = 500;
@@ -60,6 +57,7 @@ public class JobProgressPoller extends Thread {
       try {
         Thread.sleep(intervalMs);
       } catch (InterruptedException e) {
+        logger.error("Exception in JobProgressPoller while run Thread.sleep", e);
       }
     }
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java

@@ -24,6 +24,8 @@ import java.util.List;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.zeppelin.scheduler.Job.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Parallel scheduler runs submitted job concurrently.
@@ -37,6 +39,8 @@ public class ParallelScheduler implements Scheduler {
   private String name;
   private int maxConcurrency;
 
+  static Logger LOGGER = LoggerFactory.getLogger(ParallelScheduler.class);
+
   public ParallelScheduler(String name, ExecutorService executor, SchedulerListener listener,
       int maxConcurrency) {
     this.name = name;
@@ -107,6 +111,7 @@ public class ParallelScheduler implements Scheduler {
           try {
             queue.wait(500);
           } catch (InterruptedException e) {
+            LOGGER.error("Exception in MockInterpreterAngular while interpret queue.wait", e);
           }
           continue;
         }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java

@@ -17,12 +17,6 @@
 
 package org.apache.zeppelin.scheduler;
 
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
 import org.apache.thrift.TException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@@ -32,6 +26,12 @@ import org.apache.zeppelin.scheduler.Job.Status;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
 /**
  * RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
  */
@@ -67,6 +67,7 @@ public class RemoteScheduler implements Scheduler {
           try {
             queue.wait(500);
           } catch (InterruptedException e) {
+            logger.error("Exception in RemoteScheduler while run queue.wait", e);
           }
           continue;
         }
@@ -86,6 +87,8 @@ public class RemoteScheduler implements Scheduler {
           try {
             queue.wait(500);
           } catch (InterruptedException e) {
+            logger.error("Exception in RemoteScheduler while jobRunner.isJobSubmittedInRemote " +
+                "queue.wait", e);
           }
         }
       }
@@ -194,6 +197,7 @@ public class RemoteScheduler implements Scheduler {
           try {
             this.wait(interval);
           } catch (InterruptedException e) {
+            logger.error("Exception in RemoteScheduler while run this.wait", e);
           }
         }
 
@@ -251,6 +255,7 @@ public class RemoteScheduler implements Scheduler {
         return Status.ERROR;
       }
 
+      boolean broken = false;
       try {
         String statusStr = client.getStatus(job.getId());
         if ("Unknown".equals(statusStr)) {
@@ -265,6 +270,7 @@ public class RemoteScheduler implements Scheduler {
         listener.afterStatusChange(job, null, status);
         return status;
       } catch (TException e) {
+        broken = true;
         logger.error("Can't get status information", e);
         lastStatus = Status.ERROR;
         return Status.ERROR;
@@ -273,7 +279,7 @@ public class RemoteScheduler implements Scheduler {
         lastStatus = Status.ERROR;
         return Status.ERROR;
       } finally {
-        interpreterProcess.releaseClient(client);
+        interpreterProcess.releaseClient(client, broken);
       }
     }
   }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java

@@ -30,12 +30,9 @@ import org.slf4j.LoggerFactory;
 
 /**
  * TODO(moon) : add description.
- *
- * @author Leemoonsoo
- *
  */
 public class SchedulerFactory implements SchedulerListener {
-  private final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
+  private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
   ExecutorService executor;
   Map<String, Scheduler> schedulers = new LinkedHashMap<String, Scheduler>();
 
@@ -49,7 +46,7 @@ public class SchedulerFactory implements SchedulerListener {
           try {
             singleton = new SchedulerFactory();
           } catch (Exception e) {
-            e.printStackTrace();
+            logger.error(e.toString(), e);
           }
         }
       }

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerListener.java

@@ -19,9 +19,6 @@ package org.apache.zeppelin.scheduler;
 
 /**
  * TODO(moon) : add description.
- *
- * @author Leemoonsoo
- *
  */
 public interface SchedulerListener {
   public void jobStarted(Scheduler scheduler, Job job);

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java

@@ -219,9 +219,6 @@ public class RemoteInterpreterTest {
 
     intpA.close();
     intpB.close();
-
-    RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-    assertNull(process);
   }
 
   @Test
@@ -337,9 +334,6 @@ public class RemoteInterpreterTest {
 
     intpA.close();
     intpB.close();
-
-    RemoteInterpreterProcess process = intpA.getInterpreterProcess();
-    assertNull(process);
   }
 
   @Test

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java

@@ -90,6 +90,7 @@ public class MockInterpreterAngular extends Interpreter {
     try {
       Thread.sleep(500); // wait for watcher executed
     } catch (InterruptedException e) {
+      logger.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e);
     }
 
     String msg = registry.getAll(context.getNoteId()).size() + " " + Integer.toString(numWatch.get());

zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java

@@ -22,6 +22,8 @@ import java.util.Map;
 
 import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.scheduler.JobListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SleepingJob extends Job{
 
@@ -30,6 +32,8 @@ public class SleepingJob extends Job{
 	private long start;
 	private int count;
 
+	static Logger LOGGER = LoggerFactory.getLogger(SleepingJob.class);
+
 
 	public SleepingJob(String jobName, JobListener listener, int time){
 		super(jobName, listener);
@@ -44,6 +48,7 @@ public class SleepingJob extends Job{
 			try {
 				Thread.sleep(10);
 			} catch (InterruptedException e) {
+				LOGGER.error("Exception in MockInterpreterAngular while interpret Thread.sleep", e);
 			}
 			if(System.currentTimeMillis() - start>time) break;
 		}

zeppelin-server/pom.xml

@@ -269,6 +269,16 @@
       <version>1.9.0</version>
       <scope>test</scope>
     </dependency>
+
+    <!-- Apache Shiro -->
+    <dependency>
+      <groupId>org.apache.shiro</groupId>
+      <artifactId>shiro-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.shiro</groupId>
+      <artifactId>shiro-web</artifactId>
+    </dependency>
   </dependencies>
 
   <build>

zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java

@@ -110,9 +110,11 @@ public class InterpreterRestApi {
       interpreterFactory.setPropertyAndRestart(settingId,
           new InterpreterOption(true), p.getProperties());
     } catch (InterpreterException e) {
+      logger.error("Exception in InterpreterRestApi while updateSetting ", e);
       return new JsonResponse(
           Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
     } catch (IOException e) {
+      logger.error("Exception in InterpreterRestApi while updateSetting ", e);
       return new JsonResponse(
           Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
     }
@@ -144,6 +146,7 @@ public class InterpreterRestApi {
     try {
       interpreterFactory.restart(settingId);
     } catch (InterpreterException e) {
+      logger.error("Exception in InterpreterRestApi while restartSetting ", e);
       return new JsonResponse(
           Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
     }

zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookResponse.java

@@ -21,9 +21,6 @@ import javax.xml.bind.annotation.XmlRootElement;
 
 /**
  * Response wrapper.
- *
- * @author anthonycorbacho
- *
  */
 @XmlRootElement
 public class NotebookResponse {

zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java

@@ -131,7 +131,7 @@ public class NotebookRestApi {
   @GET
   @Path("/")
   public Response getNotebookList() throws IOException {
-    List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo();
+    List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false);
     return new JsonResponse<>(Status.OK, "", notesInfo ).build();
   }
 
@@ -218,7 +218,130 @@ public class NotebookRestApi {
     notebookServer.broadcastNoteList();
     return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
   }
-  
+
+  /**
+   * Insert paragraph REST API
+   * @param message - JSON containing paragraph's information
+   * @return JSON with status.OK
+   * @throws IOException
+   */
+  @POST
+  @Path("{notebookId}/paragraph")
+  public Response insertParagraph(@PathParam("notebookId") String notebookId, String message)
+      throws IOException {
+    LOG.info("insert paragraph {} {}", notebookId, message);
+
+    Note note = notebook.getNote(notebookId);
+    if (note == null) {
+      return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
+    }
+
+    NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
+
+    Paragraph p;
+    Double indexDouble = request.getIndex();
+    if (indexDouble == null) {
+      p = note.addParagraph();
+    } else {
+      p = note.insertParagraph(indexDouble.intValue());
+    }
+    p.setTitle(request.getTitle());
+    p.setText(request.getText());
+
+    note.persist();
+    notebookServer.broadcastNote(note);
+    return new JsonResponse(Status.CREATED, "", p.getId()).build();
+  }
+
+  /**
+   * Get paragraph REST API
+   * @param
+   * @return JSON with information of the paragraph
+   * @throws IOException
+   */
+  @GET
+  @Path("{notebookId}/paragraph/{paragraphId}")
+  public Response getParagraph(@PathParam("notebookId") String notebookId,
+                               @PathParam("paragraphId") String paragraphId) throws IOException {
+    LOG.info("get paragraph {} {}", notebookId, paragraphId);
+
+    Note note = notebook.getNote(notebookId);
+    if (note == null) {
+      return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
+    }
+
+    Paragraph p = note.getParagraph(paragraphId);
+    if (p == null) {
+      return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
+    }
+
+    return new JsonResponse(Status.OK, "", p).build();
+  }
+
+  /**
+   * Move paragraph REST API
+   * @param newIndex - new index to move
+   * @return JSON with status.OK
+   * @throws IOException
+   */
+  @POST
+  @Path("{notebookId}/paragraph/{paragraphId}/move/{newIndex}")
+  public Response moveParagraph(@PathParam("notebookId") String notebookId,
+                                @PathParam("paragraphId") String paragraphId,
+                                @PathParam("newIndex") String newIndex) throws IOException {
+    LOG.info("move paragraph {} {} {}", notebookId, paragraphId, newIndex);
+
+    Note note = notebook.getNote(notebookId);
+    if (note == null) {
+      return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
+    }
+
+    Paragraph p = note.getParagraph(paragraphId);
+    if (p == null) {
+      return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
+    }
+
+    try {
+      note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true);
+
+      note.persist();
+      notebookServer.broadcastNote(note);
+      return new JsonResponse(Status.OK, "").build();
+    } catch (IndexOutOfBoundsException e) {
+      LOG.error("Exception in NotebookRestApi while moveParagraph ", e);
+      return new JsonResponse(Status.BAD_REQUEST, "paragraph's new index is out of bound").build();
+    }
+  }
+
+  /**
+   * Delete paragraph REST API
+   * @param
+   * @return JSON with status.OK
+   * @throws IOException
+   */
+  @DELETE
+  @Path("{notebookId}/paragraph/{paragraphId}")
+  public Response deleteParagraph(@PathParam("notebookId") String notebookId,
+                                  @PathParam("paragraphId") String paragraphId) throws IOException {
+    LOG.info("delete paragraph {} {}", notebookId, paragraphId);
+
+    Note note = notebook.getNote(notebookId);
+    if (note == null) {
+      return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
+    }
+
+    Paragraph p = note.getParagraph(paragraphId);
+    if (p == null) {
+      return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
+    }
+
+    note.removeParagraph(paragraphId);
+    note.persist();
+    notebookServer.broadcastNote(note);
+
+    return new JsonResponse(Status.OK, "").build();
+  }
+
   /**
    * Run notebook jobs REST API
    * @param

zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.server.JsonResponse;
+import org.apache.zeppelin.ticket.TicketContainer;
+import org.apache.zeppelin.utils.SecurityUtils;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Zeppelin security rest api endpoint.
+ *
+ */
+@Path("/security")
+@Produces("application/json")
+public class SecurityRestApi {
+  /**
+   * Required by Swagger.
+   */
+  public SecurityRestApi() {
+    super();
+  }
+
+  /**
+   * Get ticket
+   * Returns username & ticket
+   * for anonymous access, username is always anonymous.
+   * After getting this ticket, access through websockets become safe
+   *
+   * @return 200 response
+   */
+  @GET
+  @Path("ticket")
+  public Response ticket() {
+    ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+    String principal = SecurityUtils.getPrincipal();
+    JsonResponse response;
+    // ticket set to anonymous for anonymous user. Simplify testing.
+    String ticket;
+    if ("anonymous".equals(principal))
+      ticket = "anonymous";
+    else
+      ticket = TicketContainer.instance.getTicket(principal);
+
+    Map<String, String> data = new HashMap<>();
+    data.put("principal", principal);
+    data.put("ticket", ticket);
+
+    response = new JsonResponse(Response.Status.OK, "", data);
+    return response.build();
+  }
+}

zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewParagraphRequest.java

@@ -20,12 +20,12 @@ package org.apache.zeppelin.rest.message;
 /**
  * NewParagraphRequest rest api request message
  *
- * It is used for NewNotebookRequest with initial paragraphs
- *
+ * index field will be ignored when it's used to provide initial paragraphs
  */
 public class NewParagraphRequest {
   String title;
   String text;
+  Double index;
 
   public NewParagraphRequest() {
 
@@ -38,4 +38,8 @@ public class NewParagraphRequest {
   public String getText() {
     return text;
   }
+
+  public Double getIndex() {
+    return index;
+  }
 }

zeppelin-server/src/main/java/org/apache/zeppelin/server/CorsFilter.java

@@ -19,12 +19,12 @@ package org.apache.zeppelin.server;
 
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.utils.SecurityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.text.DateFormat;
-import java.util.Arrays;
 import java.util.Date;
 import java.util.Locale;
 
@@ -43,6 +43,8 @@ import javax.servlet.http.HttpServletResponse;
  */
 public class CorsFilter implements Filter {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(CorsFilter.class);
+
   @Override
   public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain)
       throws IOException, ServletException {
@@ -54,7 +56,7 @@ public class CorsFilter implements Filter {
         origin = sourceHost;
       }
     } catch (URISyntaxException e) {
-      e.printStackTrace();
+      LOGGER.error("Exception in WebDriverManager while getWebDriver ", e);
     }
 
     if (((HttpServletRequest) request).getMethod().equals("OPTIONS")) {

zeppelin-server/src/main/java/org/apache/zeppelin/server/JsonResponse.java

@@ -31,8 +31,6 @@ import com.google.gson.GsonBuilder;
 /**
  * Json response builder.
  *
- * @author Leemoonsoo
- *
  * @param <T>
  */
 public class JsonResponse<T> {

zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java

@@ -36,6 +36,7 @@ import org.apache.zeppelin.notebook.repo.NotebookRepo;
 import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
 import org.apache.zeppelin.rest.InterpreterRestApi;
 import org.apache.zeppelin.rest.NotebookRestApi;
+import org.apache.zeppelin.rest.SecurityRestApi;
 import org.apache.zeppelin.rest.ZeppelinRestApi;
 import org.apache.zeppelin.scheduler.SchedulerFactory;
 import org.apache.zeppelin.search.SearchService;
@@ -135,6 +136,7 @@ public class ZeppelinServer extends Application {
       try {
         System.in.read();
       } catch (IOException e) {
+        LOG.error("Exception in ZeppelinServer while main ", e);
       }
       System.exit(0);
     }
@@ -226,6 +228,12 @@ public class ZeppelinServer extends Application {
 
     cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*",
         EnumSet.allOf(DispatcherType.class));
+
+    cxfContext.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*",
+        EnumSet.allOf(DispatcherType.class));
+
+    cxfContext.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener());
+
     return cxfContext;
   }
 
@@ -273,6 +281,9 @@ public class ZeppelinServer extends Application {
     InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory);
     singletons.add(interpreterApi);
 
+    SecurityRestApi securityApi = new SecurityRestApi();
+    singletons.add(securityApi);
+
     return singletons;
   }
 }

zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java

@@ -22,16 +22,10 @@ import java.util.Map;
 
 /**
  * Zeppelin websocker massage template class.
- *
- * @author Leemoonsoo
- *
  */
 public class Message {
   /**
    * Representation of event type.
-   *
-   * @author Leemoonsoo
-   *
    */
   public static enum OP {
     GET_HOME_NOTE, // [c-s] load note for home screen
@@ -92,6 +86,7 @@ public class Message {
                      // @param completions list of string
 
     LIST_NOTES, // [c-s] ask list of note
+    RELOAD_NOTES_FROM_REPO, // [c-s] reload notes from repo
 
     NOTES_INFO, // [s-c] list of note infos
                 // @param notes serialized List<NoteInfo> object
@@ -108,6 +103,8 @@ public class Message {
 
   public OP op;
   public Map<String, Object> data = new HashMap<String, Object>();
+  public String ticket = "anonymous";
+  public String principal = "anonymous";
 
   public Message(OP op) {
     this.op = op;

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -41,6 +41,7 @@ import org.apache.zeppelin.scheduler.Job.Status;
 import org.apache.zeppelin.scheduler.JobListener;
 import org.apache.zeppelin.server.ZeppelinServer;
 import org.apache.zeppelin.socket.Message.OP;
+import org.apache.zeppelin.ticket.TicketContainer;
 import org.apache.zeppelin.utils.SecurityUtils;
 import org.eclipse.jetty.websocket.WebSocket;
 import org.eclipse.jetty.websocket.WebSocketServlet;
@@ -71,9 +72,9 @@ public class NotebookServer extends WebSocketServlet implements
     try {
       return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create());
     } catch (UnknownHostException e) {
-      e.printStackTrace();
+      LOG.error(e.toString(), e);
     } catch (URISyntaxException e) {
-      e.printStackTrace();
+      LOG.error(e.toString(), e);
     }
     return false;
   }
@@ -96,11 +97,27 @@ public class NotebookServer extends WebSocketServlet implements
     try {
       Message messagereceived = deserializeMessage(msg);
       LOG.debug("RECEIVE << " + messagereceived.op);
+      LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal);
+      LOG.debug("RECEIVE TICKET << " + messagereceived.ticket);
+      String ticket = TicketContainer.instance.getTicket(messagereceived.principal);
+      if (ticket != null && !ticket.equals(messagereceived.ticket))
+        throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket);
+
+      ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+      boolean allowAnonymous = conf.
+          getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
+      if (!allowAnonymous && messagereceived.principal.equals("anonymous")) {
+        throw new Exception("Anonymous access not allowed ");
+      }
+
       /** Lets be elegant here */
       switch (messagereceived.op) {
           case LIST_NOTES:
             broadcastNoteList();
             break;
+          case RELOAD_NOTES_FROM_REPO:
+            broadcastReloadedNoteList();
+            break;
           case GET_HOME_NOTE:
             sendHomeNote(conn, notebook);
             break;
@@ -291,7 +308,7 @@ public class NotebookServer extends WebSocketServlet implements
     }
   }
 
-  public List<Map<String, String>> generateNotebooksInfo (){
+  public List<Map<String, String>> generateNotebooksInfo(boolean needsReload) {
     Notebook notebook = notebook();
 
     ZeppelinConfiguration conf = notebook.getConf();
@@ -299,6 +316,14 @@ public class NotebookServer extends WebSocketServlet implements
     boolean hideHomeScreenNotebookFromList = conf
             .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE);
 
+    if (needsReload) {
+      try {
+        notebook.reloadAllNotes();
+      } catch (IOException e) {
+        LOG.error("Fail to reload notes from repository");
+      }
+    }
+
     List<Note> notes = notebook.getAllNotes();
     List<Map<String, String>> notesInfo = new LinkedList<>();
     for (Note note : notes) {
@@ -321,8 +346,12 @@ public class NotebookServer extends WebSocketServlet implements
   }
 
   public void broadcastNoteList() {
+    List<Map<String, String>> notesInfo = generateNotebooksInfo(false);
+    broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
+  }
 
-    List<Map<String, String>> notesInfo = generateNotebooksInfo();
+  public void broadcastReloadedNoteList() {
+    List<Map<String, String>> notesInfo = generateNotebooksInfo(true);
     broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
   }
 
@@ -401,7 +430,8 @@ public class NotebookServer extends WebSocketServlet implements
 
     return cronUpdated;
   }
-  private void createNote(WebSocket conn, Notebook notebook, Message message) throws IOException {
+  private void createNote(NotebookSocket conn, Notebook notebook, Message message)
+      throws IOException {
     Note note = notebook.createNote();
     note.addParagraph(); // it's an empty note. so add one paragraph
     if (message != null) {
@@ -414,7 +444,7 @@ public class NotebookServer extends WebSocketServlet implements
 
     note.persist();
     addConnectionToNote(note.id(), (NotebookSocket) conn);
-    broadcastNote(note);
+    conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
     broadcastNoteList();
   }
 
@@ -458,7 +488,7 @@ public class NotebookServer extends WebSocketServlet implements
     String name = (String) fromMessage.get("name");
     Note newNote = notebook.cloneNote(noteId, name);
     addConnectionToNote(newNote.id(), (NotebookSocket) conn);
-    broadcastNote(newNote);
+    conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
     broadcastNoteList();
   }
 
@@ -756,7 +786,7 @@ public class NotebookServer extends WebSocketServlet implements
         try {
           note.persist();
         } catch (IOException e) {
-          e.printStackTrace();
+          LOG.error(e.toString(), e);
         }
       }
       notebookServer.broadcastNote(note);

zeppelin-server/src/main/java/org/apache/zeppelin/ticket/TicketContainer.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.ticket;
+
+import java.util.Calendar;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Very simple ticket container
+ * No cleanup is done, since the same user accross different devices share the same ticket
+ * The Map size is at most the number of different user names having access to a Zeppelin instance
+ */
+
+
+public class TicketContainer {
+  private static class Entry {
+    public final String ticket;
+    // lastAccessTime still unused
+    public final long lastAccessTime;
+
+    Entry(String ticket) {
+      this.ticket = ticket;
+      this.lastAccessTime = Calendar.getInstance().getTimeInMillis();
+    }
+  }
+
+  private Map<String, Entry> sessions = new ConcurrentHashMap<>();
+
+  public static final TicketContainer instance = new TicketContainer();
+
+  /**
+   * For test use
+   * @param principal
+   * @param ticket
+   * @return true if ticket assigned to principal.
+   */
+  public boolean isValid(String principal, String ticket) {
+    if ("anonymous".equals(principal) && "anonymous".equals(ticket))
+      return true;
+    Entry entry = sessions.get(principal);
+    return entry != null && entry.ticket.equals(ticket);
+  }
+
+  /**
+   * get or create ticket for Websocket authentication assigned to authenticated shiro user
+   * For unathenticated user (anonymous), always return ticket value "anonymous"
+   * @param principal
+   * @return
+   */
+  public synchronized String getTicket(String principal) {
+    Entry entry = sessions.get(principal);
+    String ticket;
+    if (entry == null) {
+      if (principal.equals("anonymous"))
+        ticket = "anonymous";
+      else
+        ticket = UUID.randomUUID().toString();
+    } else {
+      ticket = entry.ticket;
+    }
+    entry = new Entry(ticket);
+    sessions.put(principal, entry);
+    return ticket;
+  }
+}

zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java

@@ -16,6 +16,7 @@
  */
 package org.apache.zeppelin.utils;
 
+import org.apache.shiro.subject.Subject;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 
 import java.net.InetAddress;
@@ -44,4 +45,20 @@ public class SecurityUtils {
             "localhost".equals(sourceUriHost) ||
             conf.getAllowedOrigins().contains(sourceHost);
   }
+
+  /**
+   * Return the authenticated user if any otherwise returns "anonymous"
+   * @return shiro principal
+   */
+  public static String getPrincipal() {
+    Subject subject = org.apache.shiro.SecurityUtils.getSubject();
+    String principal;
+    if (subject.isAuthenticated()) {
+      principal = subject.getPrincipal().toString();
+    }
+    else {
+      principal = "anonymous";
+    }
+    return principal;
+  }
 }

zeppelin-server/src/main/resources/shiro.ini

@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[users]
+# List of users with their password allowed to access Zeppelin.
+# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
+admin = password
+
+
+[urls]
+
+# anon means the access is anonymous.
+# authcBasic means Basic Auth Security
+# To enfore security, comment the line below and uncomment the next one
+/** = anon
+#/** = authcBasic
+

zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java

@@ -50,6 +50,8 @@ import com.gargoylesoftware.htmlunit.WebRequest;
 import com.gargoylesoftware.htmlunit.WebWindow;
 import com.gargoylesoftware.htmlunit.html.HtmlElement;
 import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * from https://code.google.com/p/selenium/issues/detail?id=1361
@@ -63,6 +65,8 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
     // http://stackoverflow.com/questions/4652777/java-regex-to-get-the-urls-from-css
     private final static Pattern cssUrlPattern = Pattern.compile("background(-image)?[\\s]*:[^url]*url[\\s]*\\([\\s]*([^\\)]*)[\\s]*\\)[\\s]*");// ?<url>
 
+    static Logger LOGGER = LoggerFactory.getLogger(ScreenCaptureHtmlUnitDriver.class);
+
     public ScreenCaptureHtmlUnitDriver() {
         super();
     }
@@ -88,6 +92,7 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
         try {
             archive = downloadCssAndImages(getWebClient(), (HtmlPage) getCurrentWindow().getEnclosedPage());
         } catch (Exception e) {
+            LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while getScreenshotAs ", e);
         }
         if(target.equals(OutputType.BASE64)){
             return target.convertFromBase64Png(new Base64Encoder().encode(archive));
@@ -116,6 +121,7 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
             window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
             webClient.getPage(window, new WebRequest(page.getUrl()));
         } catch (Exception e) {
+            LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while downloadCssAndImages ", e);
             window = webClient.openWindow(page.getUrl(), page.getUrl().toString()+"_screenshot");
         }
 
@@ -148,6 +154,7 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
                             .replace("resources/", "./").getBytes());
                 }
             } catch (Exception e) {
+                LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while resultList.iterator ", e);
             }
         }
         String pagesrc = replaceRemoteUrlsWithLocal(page.getWebResponse().getContentAsString(), urlMapping);

zeppelin-server/src/test/java/org/apache/zeppelin/CommandExecutor.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class CommandExecutor {
+
+  public final static Logger LOG = LoggerFactory.getLogger(CommandExecutor.class);
+
+  public enum IGNORE_ERRORS {
+    TRUE,
+    FALSE
+  }
+
+  public static int NORMAL_EXIT = 0;
+
+  private static IGNORE_ERRORS DEFAULT_BEHAVIOUR_ON_ERRORS = IGNORE_ERRORS.TRUE;
+
+  public static Object executeCommandLocalHost(String[] command, boolean printToConsole, ProcessData.Types_Of_Data type, IGNORE_ERRORS ignore_errors) {
+    List<String> subCommandsAsList = new ArrayList<String>(Arrays.asList(command));
+    String mergedCommand = StringUtils.join(subCommandsAsList, " ");
+
+    LOG.info("Sending command \"" + mergedCommand + "\" to localhost");
+
+    ProcessBuilder processBuilder = new ProcessBuilder(command);
+    Process process = null;
+    try {
+      process = processBuilder.start();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    ProcessData data_of_process = new ProcessData(process, printToConsole);
+    Object output_of_process = data_of_process.getData(type);
+    int exit_code = data_of_process.getExitCodeValue();
+
+    if (!printToConsole)
+      LOG.trace(output_of_process.toString());
+    else
+      LOG.debug(output_of_process.toString());
+    if (ignore_errors == IGNORE_ERRORS.FALSE && exit_code != NORMAL_EXIT) {
+      LOG.error(String.format("*********************Command '%s' failed with exitcode %s *********************", mergedCommand, exit_code));
+    }
+    return output_of_process;
+  }
+
+  public static Object executeCommandLocalHost(String command, boolean printToConsole, ProcessData.Types_Of_Data type) {
+    return executeCommandLocalHost(new String[]{"bash", "-c", command}, printToConsole, type, DEFAULT_BEHAVIOUR_ON_ERRORS);
+  }
+
+}

zeppelin-server/src/test/java/org/apache/zeppelin/ProcessData.java

@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.concurrent.TimeUnit;
+
+public class ProcessData {
+  public enum Types_Of_Data {
+    OUTPUT,
+    ERROR,
+    EXIT_CODE,
+    STREAMS_MERGED,
+    PROCESS_DATA_OBJECT
+  }
+
+  public final static Logger LOG = LoggerFactory.getLogger(ProcessData.class);
+
+  private Process checked_process;
+  private boolean printToConsole = false;
+  private boolean removeRedundantOutput = true;
+
+  public ProcessData(Process connected_process, boolean printToConsole, int silenceTimeout, TimeUnit timeUnit) {
+    this.checked_process = connected_process;
+    this.printToConsole = printToConsole;
+    this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeout, timeUnit);
+  }
+
+  public ProcessData(Process connected_process, boolean printToConsole, int silenceTimeoutSec) {
+    this.checked_process = connected_process;
+    this.printToConsole = printToConsole;
+    this.silenceTimeout = TimeUnit.MILLISECONDS.convert(silenceTimeoutSec, TimeUnit.SECONDS);
+  }
+
+  public ProcessData(Process connected_process, boolean printToConsole) {
+    this.checked_process = connected_process;
+    this.printToConsole = printToConsole;
+  }
+
+  public ProcessData(Process connected_process) {
+    this.checked_process = connected_process;
+    this.printToConsole = true;
+  }
+
+
+  boolean returnCodeRetrieved = false;
+
+  private String outPutStream = null;
+  private String errorStream = null;
+  private int returnCode;
+  private long silenceTimeout = 10 * 60 * 1000;
+  private final long unconditionalExitDelayMinutes = 30;
+
+  public static boolean isRunning(Process process) {
+    try {
+      process.exitValue();
+      return false;
+    } catch (IllegalThreadStateException e) {
+      return true;
+    }
+  }
+
+  public Object getData(Types_Of_Data type) {
+    //TODO get rid of Pseudo-terminal will not be allocated because stdin is not a terminal.
+    switch (type) {
+      case OUTPUT: {
+        return this.getOutPutStream();
+      }
+      case ERROR: {
+        return this.getErrorStream();
+      }
+      case EXIT_CODE: {
+        return this.getExitCodeValue();
+      }
+      case STREAMS_MERGED: {
+        return this.getOutPutStream() + "\n" + this.getErrorStream();
+      }
+      case PROCESS_DATA_OBJECT: {
+        this.getErrorStream();
+        return this;
+      }
+      default: {
+        throw new IllegalArgumentException("Data Type " + type + " not supported yet!");
+      }
+    }
+  }
+
+  public int getExitCodeValue() {
+    try {
+      if (!returnCodeRetrieved) {
+        this.checked_process.waitFor();
+        this.returnCode = this.checked_process.exitValue();
+        this.returnCodeRetrieved = true;
+        this.checked_process.destroy();
+      }
+    } catch (Exception inter) {
+      throw new RuntimeException("Couldn't finish waiting for process " + this.checked_process + " termination", inter);
+    }
+    return this.returnCode;
+  }
+
+  public String getOutPutStream() {
+    if (this.outPutStream == null) {
+      try {
+        buildOutputAndErrorStreamData();
+      } catch (Exception e) {
+        throw new RuntimeException("Couldn't retrieve Output Stream data from process: " + this.checked_process.toString(), e);
+
+      }
+    }
+    this.outPutStream = this.outPutStream.replace("Pseudo-terminal will not be allocated because stdin is not a terminal.", "");
+    this.errorStream = this.errorStream.replace("Pseudo-terminal will not be allocated because stdin is not a terminal.", "");
+    return this.outPutStream;
+  }
+
+  public String getErrorStream() {
+    if (this.errorStream == null) {
+      try {
+        buildOutputAndErrorStreamData();
+      } catch (Exception e) {
+        throw new RuntimeException("Couldn't retrieve Error Stream data from process: " + this.checked_process.toString(), e);
+
+      }
+    }
+    this.outPutStream = this.outPutStream.replace("Pseudo-terminal will not be allocated because stdin is not a terminal.", "");
+    this.errorStream = this.errorStream.replace("Pseudo-terminal will not be allocated because stdin is not a terminal.", "");
+    return this.errorStream;
+  }
+
+  public String toString() {
+    StringBuilder result = new StringBuilder();
+    result.append(String.format("[OUTPUT STREAM]\n%s\n", this.outPutStream));
+    result.append(String.format("[ERROR STREAM]\n%s\n", this.errorStream));
+    result.append(String.format("[EXIT CODE]\n%d", this.returnCode));
+    return result.toString();
+  }
+
+  private void buildOutputAndErrorStreamData() throws IOException {
+    StringBuilder sbInStream = new StringBuilder();
+    StringBuilder sbErrorStream = new StringBuilder();
+
+    try {
+      InputStream in = this.checked_process.getInputStream();
+      InputStream inErrors = this.checked_process.getErrorStream();
+      BufferedReader inReader = new BufferedReader(new InputStreamReader(in));
+      BufferedReader inReaderErrors = new BufferedReader(new InputStreamReader(inErrors));
+      LOG.trace("Started retrieving data from streams of attached process: " + this.checked_process);
+
+      long lastStreamDataTime = System.currentTimeMillis();   //Store start time to be able to finish method if command hangs
+      long unconditionalExitTime = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(unconditionalExitDelayMinutes, TimeUnit.MINUTES); // Stop after 'unconditionalExitDelayMinutes' even if process is alive and sending output
+      final int BUFFER_LEN = 300;
+      char charBuffer[] = new char[BUFFER_LEN];     //Use char buffer to read output, size can be tuned.
+      boolean outputProduced = true;                //Flag to check if previous iteration produced any output
+      while (isRunning(this.checked_process) || outputProduced) {   //Continue if process is alive or some output was produced on previous iteration and there may be still some data to read.
+        outputProduced = false;
+        ZeppelinITUtils.sleep(100, false);                                  //Some local commands can exit fast, but immediate stream reading will give no output and after iteration, 'while' condition will be false so we will not read out any output while it is still there, just need to wait for some time for it to appear in streams.
+
+        StringBuilder tempSB = new StringBuilder();
+        while (inReader.ready()) {
+          tempSB.setLength(0);                                // clean temporary StringBuilder
+          int readCount = inReader.read(charBuffer, 0, BUFFER_LEN); //read up to 'BUFFER_LEN' chars to buffer
+          if (readCount < 1) {                                     // if nothing read or error occurred
+            break;
+          }
+          tempSB.append(charBuffer, 0, readCount);
+
+          sbInStream.append(tempSB);
+          if (tempSB.length() > 0) {
+            outputProduced = true;                                //set flag to know that we read something and there may be moire data, even if process already exited
+          }
+
+          lastStreamDataTime = System.currentTimeMillis();        //remember last time data was read from streams to be sure we are not looping infinitely
+        }
+
+        tempSB = new StringBuilder();                               //Same, but for error stream
+        while (inReaderErrors.ready()) {
+          tempSB.setLength(0);
+          int readCount = inReaderErrors.read(charBuffer, 0, BUFFER_LEN);
+          if (readCount < 1) {
+            break;
+          }
+          tempSB.append(charBuffer, 0, readCount);
+          sbErrorStream.append(tempSB);
+          if (tempSB.length() > 0) {
+            outputProduced = true;
+            String temp = new String(tempSB);
+            temp = temp.replaceAll("Pseudo-terminal will not be allocated because stdin is not a terminal.", "");
+            //TODO : error stream output need to be improved, because it outputs downloading information.
+            if (printToConsole) {
+              if (!temp.trim().equals("")) {
+                if (temp.toLowerCase().contains("error") || temp.toLowerCase().contains("failed")) {
+                  LOG.warn(temp.trim());
+                } else {
+                  LOG.debug(temp.trim());
+                }
+              }
+            }
+          }
+          lastStreamDataTime = System.currentTimeMillis();
+        }
+
+
+        if ((System.currentTimeMillis() - lastStreamDataTime > silenceTimeout) ||     //Exit if silenceTimeout ms has passed from last stream read. Means process is alive but not sending any data.
+            (System.currentTimeMillis() > unconditionalExitTime)) {                    //Exit unconditionally - guards against alive process continuously sending data.
+          LOG.info("Conditions: " + (System.currentTimeMillis() - lastStreamDataTime > silenceTimeout) + " " +
+              (System.currentTimeMillis() > unconditionalExitTime));
+          this.checked_process.destroy();
+          try {
+            if ((System.currentTimeMillis() > unconditionalExitTime))
+              LOG.error("!@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@Unconditional exit occured@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@!\nsome process hag up for more than " + unconditionalExitDelayMinutes + " minutes.");
+            LOG.error("!##################################!");
+            StringWriter sw = new StringWriter();
+            new Exception("Exited from buildOutputAndErrorStreamData by timeout").printStackTrace(new PrintWriter(sw)); //Get stack trace
+            String exceptionAsString = sw.toString();
+            LOG.error(exceptionAsString);
+          } catch (Exception ignore) {
+            LOG.info("Exception in ProcessData while buildOutputAndErrorStreamData ", ignore);
+          }
+          break;
+        }
+      }
+
+      in.close();
+      inErrors.close();
+    } finally {
+      this.outPutStream = sbInStream.toString();
+      this.errorStream = sbErrorStream.toString();
+    }
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java

@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin;
+
+import org.apache.commons.io.FileUtils;
+import org.openqa.selenium.By;
+import org.openqa.selenium.TimeoutException;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.firefox.FirefoxBinary;
+import org.openqa.selenium.firefox.FirefoxDriver;
+import org.openqa.selenium.firefox.FirefoxProfile;
+import org.openqa.selenium.safari.SafariDriver;
+import org.openqa.selenium.support.ui.ExpectedCondition;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import static org.junit.Assert.fail;
+
+
+public class WebDriverManager {
+
+  public final static Logger LOG = LoggerFactory.getLogger(WebDriverManager.class);
+
+  private static String downLoadsDir = "";
+
+  static WebDriver getWebDriver() {
+    WebDriver driver = null;
+
+    if (driver == null) {
+      try {
+        FirefoxBinary ffox = new FirefoxBinary();
+        if ("true".equals(System.getenv("TRAVIS"))) {
+          ffox.setEnvironmentProperty("DISPLAY", ":99"); // xvfb is supposed to
+          // run with DISPLAY 99
+        }
+        int firefoxVersion = WebDriverManager.getFirefoxVersion();
+        LOG.info("Firefox version " + firefoxVersion + " detected");
+
+        downLoadsDir = FileUtils.getTempDirectory().toString();
+
+        String tempPath = downLoadsDir + "/firebug/";
+
+        downloadFireBug(firefoxVersion, tempPath);
+
+        final String firebugPath = tempPath + "firebug.xpi";
+        final String firepathPath = tempPath + "firepath.xpi";
+
+        FirefoxProfile profile = new FirefoxProfile();
+        profile.setPreference("browser.download.folderList", 2);
+        profile.setPreference("browser.download.dir", downLoadsDir);
+        profile.setPreference("browser.helperApps.alwaysAsk.force", false);
+        profile.setPreference("browser.download.manager.showWhenStarting", false);
+        profile.setPreference("browser.download.manager.showAlertOnComplete", false);
+        profile.setPreference("browser.download.manager.closeWhenDone", true);
+        profile.setPreference("app.update.auto", false);
+        profile.setPreference("app.update.enabled", false);
+        profile.setPreference("dom.max_script_run_time", 0);
+        profile.setPreference("dom.max_chrome_script_run_time", 0);
+        profile.setPreference("browser.helperApps.neverAsk.saveToDisk", "application/x-ustar,application/octet-stream,application/zip,text/csv,text/plain");
+        profile.setPreference("network.proxy.type", 0);
+
+        profile.addExtension(new File(firebugPath));
+        profile.addExtension(new File(firepathPath));
+
+        driver = new FirefoxDriver(ffox, profile);
+      } catch (Exception e) {
+        LOG.error("Exception in WebDriverManager while FireFox Driver ", e);
+      }
+    }
+
+    if (driver == null) {
+      try {
+        driver = new ChromeDriver();
+      } catch (Exception e) {
+        LOG.error("Exception in WebDriverManager while ChromeDriver ", e);
+      }
+    }
+
+    if (driver == null) {
+      try {
+        driver = new SafariDriver();
+      } catch (Exception e) {
+        LOG.error("Exception in WebDriverManager while SafariDriver ", e);
+      }
+    }
+
+    String url;
+    if (System.getProperty("url") != null) {
+      url = System.getProperty("url");
+    } else {
+      url = "http://localhost:8080";
+    }
+
+    long start = System.currentTimeMillis();
+    boolean loaded = false;
+    driver.get(url);
+
+    while (System.currentTimeMillis() - start < 60 * 1000) {
+      // wait for page load
+      try {
+        (new WebDriverWait(driver, 5)).until(new ExpectedCondition<Boolean>() {
+          @Override
+          public Boolean apply(WebDriver d) {
+            return d.findElement(By.partialLinkText("Create new note"))
+                .isDisplayed();
+          }
+        });
+        loaded = true;
+        break;
+      } catch (TimeoutException e) {
+        LOG.info("Exception in WebDriverManager while WebDriverWait ", e);
+        driver.navigate().to(url);
+      }
+    }
+
+    if (loaded == false) {
+      fail();
+    }
+
+    return driver;
+  }
+
+  private static void downloadFireBug(int firefoxVersion, String tempPath) {
+    String firebugUrlString = null;
+    if (firefoxVersion < 23)
+      firebugUrlString = "http://getfirebug.com/releases/firebug/1.11/firebug-1.11.4.xpi";
+    else if (firefoxVersion >= 23 && firefoxVersion < 30)
+      firebugUrlString = "http://getfirebug.com/releases/firebug/1.12/firebug-1.12.8.xpi";
+    else if (firefoxVersion >= 30)
+      firebugUrlString = "http://getfirebug.com/releases/firebug/2.0/firebug-2.0.7.xpi";
+
+
+    LOG.info("firebug version: " + firefoxVersion + ", will be downloaded to " + tempPath);
+    try {
+      File firebugFile = new File(tempPath + "firebug.xpi");
+      URL firebugUrl = new URL(firebugUrlString);
+      if (!firebugFile.exists()) {
+        FileUtils.copyURLToFile(firebugUrl, firebugFile);
+      }
+
+
+      File firepathFile = new File(tempPath + "firepath.xpi");
+      URL firepathUrl = new URL("https://addons.cdn.mozilla.net/user-media/addons/11900/firepath-0.9.7.1-fx.xpi");
+      if (!firepathFile.exists()) {
+        FileUtils.copyURLToFile(firepathUrl, firepathFile);
+      }
+
+    } catch (IOException e) {
+      LOG.error("Download of firebug version: " + firefoxVersion + ", falied in path " + tempPath);
+    }
+    LOG.info("Download of firebug version: " + firefoxVersion + ", successful");
+  }
+
+  public static int getFirefoxVersion() {
+    try {
+      String firefoxVersionCmd = "firefox -v";
+      if (System.getProperty("os.name").startsWith("Mac OS")) {
+        firefoxVersionCmd = "/Applications/Firefox.app/Contents/MacOS/" + firefoxVersionCmd;
+      }
+      String versionString = (String) CommandExecutor.executeCommandLocalHost(firefoxVersionCmd, false, ProcessData.Types_Of_Data.OUTPUT);
+      return Integer.valueOf(versionString.replaceAll("Mozilla Firefox", "").trim().substring(0, 2));
+    } catch (Exception e) {
+      LOG.error("Exception in WebDriverManager while getWebDriver ", e);
+      return -1;
+    }
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java

@@ -68,70 +68,12 @@ public class ZeppelinIT {
   private static final long MAX_PARAGRAPH_TIMEOUT_SEC = 60;
   private WebDriver driver;
 
-  private void setWebDriver() {
-
-    if (driver == null) {
-      try {
-        FirefoxBinary ffox = new FirefoxBinary();
-        if ("true".equals(System.getenv("TRAVIS"))) {
-          ffox.setEnvironmentProperty("DISPLAY", ":99"); // xvfb is supposed to
-                                                         // run with DISPLAY 99
-        }
-        FirefoxProfile profile = new FirefoxProfile();
-        driver = new FirefoxDriver(ffox, profile);
-      } catch (Exception e) {
-        LOG.error("Starting Firefox failed",e);
-      }
-    }
-
-    if (driver == null) {
-      try {
-        driver = new ChromeDriver();
-      } catch (Exception e) {
-        LOG.error("Starting Chrome failed",e);
-      }
-    }
-
-    if (driver == null) {
-      try {
-        driver = new SafariDriver();
-      } catch (Exception e) {
-        LOG.error("Starting Safari failed",e);
-      }
-    }
-
-    String url;
-    if (System.getProperty("url") != null) {
-      url = System.getProperty("url");
-    } else {
-      url = "http://localhost:8080";
-    }
-
-    long start = System.currentTimeMillis();
-    boolean loaded = false;
-    driver.get(url);
-
-    while (System.currentTimeMillis() - start < 60 * 1000) {
-      try { // wait for page load
-        WebElement element = pollingWait(By.partialLinkText("Create new note"), MAX_BROWSER_TIMEOUT_SEC);
-        loaded = element.isDisplayed();
-        break;
-      } catch (TimeoutException e) {
-        driver.navigate().to(url);
-      }
-    }
-
-    if (loaded == false) {
-      fail();
-    }
-  }
-
   @Before
   public void startUp() {
     if (!endToEndTestEnabled()) {
       return;
     }
-    setWebDriver();
+    driver = WebDriverManager.getWebDriver();
   }
 
   @After
@@ -159,6 +101,7 @@ public class ZeppelinIT {
       WebElement element = pollingWait(locator, MAX_BROWSER_TIMEOUT_SEC);
       return txt.equals(element.getText());
     } catch (TimeoutException e) {
+      LOG.error("Exception in ZeppelinIT while waitForText ", e);
       return false;
     }
   }
@@ -186,126 +129,134 @@ public class ZeppelinIT {
       return;
     }
     try {
-    createNewNote();
+      createNewNote();
 
-    // wait for first paragraph's " READY " status text
-    waitForParagraph(1, "READY");
+      // wait for first paragraph's " READY " status text
+      waitForParagraph(1, "READY");
 
-    /*
-     * print angular template
-     * %angular <div id='angularTestButton' ng-click='myVar=myVar+1'>BindingTest_{{myVar}}_</div>
-     */
-    WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
-    paragraph1Editor.sendKeys("println" + Keys.chord(Keys.SHIFT, "9") + "\""
-                + Keys.chord(Keys.SHIFT, "5")
-                + "angular <div id='angularTestButton' "
-                + "ng" + Keys.chord(Keys.SUBTRACT) + "click='myVar=myVar+1'>"
-                + "BindingTest_{{myVar}}_</div>\")");
-    paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(1, "FINISHED");
+      /*
+       * print angular template
+       * %angular <div id='angularTestButton' ng-click='myVar=myVar+1'>BindingTest_{{myVar}}_</div>
+       */
+      WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+      paragraph1Editor.sendKeys("println" + Keys.chord(Keys.SHIFT, "9") + "\""
+                  + Keys.chord(Keys.SHIFT, "5")
+                  + "angular <div id='angularTestButton' "
+                  + "ng" + Keys.chord(Keys.SUBTRACT) + "click='myVar=myVar+1'>"
+                  + "BindingTest_{{myVar}}_</div>\")");
+      paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(1, "FINISHED");
 
-    // check expected text
-    waitForText("BindingTest__", By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      // check expected text
+      waitForText("BindingTest__", By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
 
-    /*
-     * Bind variable
-     * z.angularBind("myVar", 1)
-     */
-    assertEquals(1, driver.findElements(By.xpath(getParagraphXPath(2) + "//textarea")).size());
-    WebElement paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
-    paragraph2Editor.sendKeys("z.angularBind" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\", 1)");
-    paragraph2Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(2, "FINISHED");
+      /*
+       * Bind variable
+       * z.angularBind("myVar", 1)
+       */
+      assertEquals(1, driver.findElements(By.xpath(getParagraphXPath(2) + "//textarea")).size());
+      WebElement paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
+      paragraph2Editor.sendKeys("z.angularBind" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\", 1)");
+      paragraph2Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(2, "FINISHED");
 
-    // check expected text
-    waitForText("BindingTest_1_", By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      // check expected text
+      waitForText("BindingTest_1_", By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
 
 
-    /*
-     * print variable
-     * print("myVar="+z.angular("myVar"))
-     */
-    WebElement paragraph3Editor = driver.findElement(By.xpath(getParagraphXPath(3) + "//textarea"));
-    paragraph3Editor.sendKeys(
-        "print" + Keys.chord(Keys.SHIFT, "9") + "\"myVar=\"" + Keys.chord(Keys.ADD)
-        + "z.angular" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\"))");
-    paragraph3Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(3, "FINISHED");
+      /*
+       * print variable
+       * print("myVar="+z.angular("myVar"))
+       */
+      WebElement paragraph3Editor = driver.findElement(By.xpath(getParagraphXPath(3) + "//textarea"));
+      paragraph3Editor.sendKeys(
+          "print" + Keys.chord(Keys.SHIFT, "9") + "\"myVar=\"" + Keys.chord(Keys.ADD)
+          + "z.angular" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\"))");
+      paragraph3Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(3, "FINISHED");
 
-    // check expected text
-    waitForText("myVar=1", By.xpath(
-        getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
+      // check expected text
+      waitForText("myVar=1", By.xpath(
+          getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
 
-    /*
-     * Click element
-     */
-    driver.findElement(By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]")).click();
+      /*
+       * Click element
+       */
+      driver.findElement(By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]")).click();
 
-    // check expected text
-    waitForText("BindingTest_2_", By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      // check expected text
+      waitForText("BindingTest_2_", By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
 
-    /*
-     * Register watcher
-     * z.angularWatch("myVar", (before:Object, after:Object, context:org.apache.zeppelin.interpreter.InterpreterContext) => {
-     *   z.run(2, context)
-     * }
-     */
-    WebElement paragraph4Editor = driver.findElement(By.xpath(getParagraphXPath(4) + "//textarea"));
-    paragraph4Editor.sendKeys(
-        "z.angularWatch" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\", "
-        + Keys.chord(Keys.SHIFT, "9")
-        + "before:Object, after:Object, context:org.apache.zeppelin.interpreter.InterpreterContext)"
-        + Keys.EQUALS + ">{ z.run" +Keys.chord(Keys.SHIFT, "9") + "2, context)}");
-    paragraph4Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(4, "FINISHED");
+      /*
+       * Register watcher
+       * z.angularWatch("myVar", (before:Object, after:Object, context:org.apache.zeppelin.interpreter.InterpreterContext) => {
+       *   z.run(2, context)
+       * }
+       */
+      WebElement paragraph4Editor = driver.findElement(By.xpath(getParagraphXPath(4) + "//textarea"));
+      paragraph4Editor.sendKeys(
+          "z.angularWatch" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\", "
+          + Keys.chord(Keys.SHIFT, "9")
+          + "before:Object, after:Object, context:org.apache.zeppelin.interpreter.InterpreterContext)"
+          + Keys.EQUALS + ">{ z.run" +Keys.chord(Keys.SHIFT, "9") + "2, context)}");
+      paragraph4Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(4, "FINISHED");
 
 
-    /*
-     * Click element, again and see watcher works
-     */
-    driver.findElement(By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]")).click();
+      /*
+       * Click element, again and see watcher works
+       */
+      driver.findElement(By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]")).click();
 
-    // check expected text
-    waitForText("BindingTest_3_", By.xpath(
-        getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
-    waitForParagraph(3, "FINISHED");
+      // check expected text
+      waitForText("BindingTest_3_", By.xpath(
+          getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      waitForParagraph(3, "FINISHED");
 
-    // check expected text by watcher
-    waitForText("myVar=3", By.xpath(
-        getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
+      // check expected text by watcher
+      waitForText("myVar=3", By.xpath(
+          getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
 
-    /*
-     * Unbind
-     * z.angularUnbind("myVar")
-     */
-    WebElement paragraph5Editor = driver.findElement(By.xpath(getParagraphXPath(5) + "//textarea"));
-    paragraph5Editor.sendKeys(
-        "z.angularUnbind" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\")");
-    paragraph5Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(5, "FINISHED");
+      /*
+       * Unbind
+       * z.angularUnbind("myVar")
+       */
+      WebElement paragraph5Editor = driver.findElement(By.xpath(getParagraphXPath(5) + "//textarea"));
+      paragraph5Editor.sendKeys(
+          "z.angularUnbind" + Keys.chord(Keys.SHIFT, "9") + "\"myVar\")");
+      paragraph5Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(5, "FINISHED");
 
-    // check expected text
-    waitForText("BindingTest__",
-        By.xpath(getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      // check expected text
+      waitForText("BindingTest__",
+          By.xpath(getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
 
-    /*
-     * Bind again and see rebind works.
-     */
-    paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
-    paragraph2Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
-    waitForParagraph(2, "FINISHED");
+      /*
+       * Bind again and see rebind works.
+       */
+      paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea"));
+      paragraph2Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+      waitForParagraph(2, "FINISHED");
 
-    // check expected text
-    waitForText("BindingTest_1_",
-        By.xpath(getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
+      // check expected text
+      waitForText("BindingTest_1_",
+          By.xpath(getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]"));
 
-    System.out.println("testCreateNotebook Test executed");
+      driver.findElement(By.xpath("//*[@id='main']/div//h3/span[1]/button[@tooltip='Remove the notebook']"))
+          .sendKeys(Keys.ENTER);
+      ZeppelinITUtils.sleep(1000, true);
+      driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this notebook')]" +
+          "//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
+      ZeppelinITUtils.sleep(100, true);
+
+      System.out.println("testCreateNotebook Test executed");
     } catch (ElementNotVisibleException e) {
+      LOG.error("Exception in ZeppelinIT while testAngularDisplay ", e);
       File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
 
     }
@@ -330,6 +281,7 @@ public class ZeppelinIT {
     try {
       Thread.sleep(500); // wait for notebook list updated
     } catch (InterruptedException e) {
+      LOG.error("Exception in ZeppelinIT while createNewNote Thread.sleep", e);
     }
   }
 }

zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinITUtils.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZeppelinITUtils {
+
+  public final static Logger LOG = LoggerFactory.getLogger(ZeppelinITUtils.class);
+
+  public static void sleep(long millis, boolean logOutput) {
+    if (logOutput) {
+      LOG.info("Starting sleeping for " + (millis / 1000) + " seconds...");
+      LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]);
+    }
+    try {
+      Thread.sleep(millis);
+    } catch (InterruptedException e) {
+      LOG.error("Exception in WebDriverManager while getWebDriver ", e);
+    }
+    if (logOutput) {
+      LOG.info("Finished.");
+    }
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java

@@ -55,7 +55,7 @@ public abstract class AbstractTestRestApi {
 
   static final String restApiUrl = "/api";
   static final String url = getUrlToTest();
-  protected static final boolean wasRunning = checkIfServerIsRuning();
+  protected static final boolean wasRunning = checkIfServerIsRunning();
   static boolean pySpark = false;
 
   private String getUrl(String path) {
@@ -86,7 +86,7 @@ public abstract class AbstractTestRestApi {
       try {
         ZeppelinServer.main(new String[] {""});
       } catch (Exception e) {
-        e.printStackTrace();
+        LOG.error("Exception in WebDriverManager while getWebDriver ", e);
         throw new RuntimeException(e);
       }
     }
@@ -101,7 +101,7 @@ public abstract class AbstractTestRestApi {
       boolean started = false;
       while (System.currentTimeMillis() - s < 1000 * 60 * 3) {  // 3 minutes
         Thread.sleep(2000);
-        started = checkIfServerIsRuning();
+        started = checkIfServerIsRunning();
         if (started == true) {
           break;
         }
@@ -156,7 +156,7 @@ public abstract class AbstractTestRestApi {
     try {
       return InetAddress.getLocalHost().getHostName();
     } catch (UnknownHostException e) {
-      e.printStackTrace();
+      LOG.error("Exception in WebDriverManager while getWebDriver ", e);
       return "localhost";
     }
   }
@@ -218,7 +218,7 @@ public abstract class AbstractTestRestApi {
       boolean started = true;
       while (System.currentTimeMillis() - s < 1000 * 60 * 3) {  // 3 minutes
         Thread.sleep(2000);
-        started = checkIfServerIsRuning();
+        started = checkIfServerIsRunning();
         if (started == false) {
           break;
         }
@@ -231,13 +231,14 @@ public abstract class AbstractTestRestApi {
     }
   }
 
-  protected static boolean checkIfServerIsRuning() {
+  protected static boolean checkIfServerIsRunning() {
     GetMethod request = null;
     boolean isRunning = true;
     try {
       request = httpGet("/");
       isRunning = request.getStatusCode() == 200;
     } catch (IOException e) {
+      LOG.error("Exception in AbstractTestRestApi while checkIfServerIsRunning ", e);
       isRunning = false;
     } finally {
       if (request != null) {
@@ -343,6 +344,7 @@ public abstract class AbstractTestRestApi {
         try {
           new JsonParser().parse(body);
         } catch (JsonParseException e) {
+          LOG.error("Exception in AbstractTestRestApi while matchesSafely ", e);
           isValid = false;
         }
         return isValid;

zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+public class SecurityRestApiTest extends AbstractTestRestApi {
+  Gson gson = new Gson();
+
+  @BeforeClass
+  public static void init() throws Exception {
+    AbstractTestRestApi.startUp();
+  }
+
+  @AfterClass
+  public static void destroy() throws Exception {
+    AbstractTestRestApi.shutDown();
+  }
+
+  @Test
+  public void testTicket() throws IOException {
+    GetMethod get = httpGet("/security/ticket");
+    get.addRequestHeader("Origin", "http://localhost");
+    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(),
+        new TypeToken<Map<String, Object>>(){}.getType());
+    Map<String, String> body = (Map<String, String>) resp.get("body");
+    assertEquals("anonymous", body.get("principal"));
+    assertEquals("anonymous", body.get("ticket"));
+    get.releaseConnection();
+  }
+
+}
+