Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #9 from apache/master

Browse source 
merge latest master updates
This commit is contained in:
mfelgamal 2016-09-07 15:40:10 +03:00 committed by GitHub
commit 852c3d3f7d
68 changed files with 1414 additions and 1022 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -47,6 +47,7 @@ zeppelin-web/bower_components
# R
/r/lib/
.Rhistory
/R/
# project level
/logs/
@ -108,3 +109,6 @@ tramp
# Generated by zeppelin-examples
/helium
# tmp files
/tmp/

26
.travis.yml
View file

@ -21,6 +21,7 @@ cache:
directories:
- .spark-dist
- ${HOME}/.m2/repository/.cache/maven-download-plugin
- .node_modules
addons:
apt:
@ -33,44 +34,49 @@ addons:
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.0.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.1
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- mkdir -p ~/R
- echo 'R_LIBS=~/R' > ~/.Renviron
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
@ -89,6 +95,7 @@ before_script:
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
- rm -rf .node_modules; cp -r zeppelin-web/node_modules .node_modules
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
@ -104,3 +111,4 @@ after_failure:
after_script:
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER

7
bin/common.cmd
View file

@ -81,13 +81,6 @@ if not defined JAVA_OPTS (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_JAVA_OPTS (
set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_MEM (
set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM%
)
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%

9
bin/common.sh
View file

@ -121,15 +121,6 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
# jvm options for interpreter process
if [[ -z "${ZEPPELIN_INTP_JAVA_OPTS}" ]]; then
export ZEPPELIN_INTP_JAVA_OPTS="${ZEPPELIN_JAVA_OPTS}"
fi
if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
export ZEPPELIN_INTP_MEM="${ZEPPELIN_MEM}"
fi
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_INTP_OPTS

4
conf/zeppelin-env.cmd.template
View file

@ -20,8 +20,8 @@ REM set JAVA_HOME=
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options.
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.

4
conf/zeppelin-env.sh.template
View file

@ -20,8 +20,8 @@
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options.
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.

3
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -105,7 +105,8 @@
<li class="title"><span><b>Advanced</b><span></li>
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Contibute</b><span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>

3
docs/assets/themes/zeppelin/css/style.css
View file

@ -438,11 +438,10 @@ a.anchor {
.content table {
display: block;
width: 100%;
overflow: auto;
word-break: normal;
word-break: keep-all;
-webkit-overflow-scrolling: touch;
font-size: 90%;
font-size: 87%;
margin-top: 16px;
margin-bottom: 16px;
}

BIN
docs/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 76 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 125 KiB

6
docs/assets/themes/zeppelin/js/search.js
View file

@ -25,7 +25,7 @@ jQuery(function() {
this.field('group');
});
window.data = $.getJSON('/search_data.json');
window.data = $.getJSON('search_data.json');
window.data.then(function(loaded_data){
$.each(loaded_data, function(index, value){
window.idx.add(
@ -51,6 +51,8 @@ jQuery(function() {
function display_search_results(results) {
var $search_results = $("#search_results");
var zeppelin_version = {{site.ZEPPELIN_VERSION | jsonify}};
var base_url = {{site.JB.BASE_PATH | jsonify}};
var prod_url = {{site.production_url | jsonify}};
window.data.then(function(loaded_data) {
if (results.length) {
@ -59,7 +61,7 @@ jQuery(function() {
results.forEach(function(result) {
var item = loaded_data[result.ref];
var appendString = '<a href="'+item.url+'">'+item.title+'</a><div class="link">'+'https://zeppelin.apache.org/docs/'+zeppelin_version+item.url+'</div><p>'+item.excerpt+'</p><br/>';
var appendString = '<a href="'+base_url+item.url.trim()+'">'+item.title+'</a><div class="link">'+prod_url+base_url+item.url.trim()+'</div><p>'+item.excerpt+'</p><br/>';
$search_results.append(appendString);
});

3
docs/index.md
View file

@ -170,7 +170,8 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* Advanced
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-yarn-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-on-yarn-mode)
* [Zeppelin on Spark Cluster Mode (Mesos via Docker)](./install/spark_cluster_mode.html#spark-on-mesos-mode)
* Contribute
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)

8
docs/install/install.md
View file

@ -195,10 +195,10 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<table class="table-configuration">
<tr>
<th>zepplin-env.sh</th>
<th>zepplin-site.xml</th>
<th>zeppelin-env.sh</th>
<th>zeppelin-site.xml</th>
<th>Default value</th>
<th>Description</th>
<th class="col-md-4">Description</th>
</tr>
<tr>
<td>ZEPPELIN_PORT</td>
@ -354,7 +354,7 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=&lt;accountName&gt;;AccountKey=&lt;accountKey&gt;</code></td>
<td>The Azure storage account connection string<br />i.e. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=&lt;accountName&gt;;<br/>AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>

66
docs/install/spark_cluster_mode.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Apache Zeppelin on Spark cluster mode"
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using docker scripts."
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager(Standalone, Hadoop Yarn, Apache Mesos) with Apache Zeppelin using docker scripts."
group: install
---
<!--
@ -113,7 +113,7 @@ docker run -it \
### 3. Verify running Spark on YARN.
You can simply verify the processes of Spark and YARN is running well in Docker with below command.
You can simply verify the processes of Spark and YARN are running well in Docker with below command.
```
ps -ef
@ -140,3 +140,65 @@ Don't forget to set Spark `master` as `yarn-client` in Zeppelin **Interpreters**
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/yarn_applications.png" />
## Spark on Mesos mode
You can simply set up [Spark on Mesos](http://spark.apache.org/docs/latest/running-on-mesos.html) docker environment with below steps.
### 1. Build Docker file
```
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_mesos
docker build -t "spark_mesos" .
```
### 2. Run docker
```
docker run --net=host -it \
-p 8080:8080 \
-p 7077:7077 \
-p 8888:8888 \
-p 8081:8081 \
-p 8082:8082 \
-p 5050:5050 \
-p 5051:5051 \
-p 4040:4040 \
-h sparkmaster \
--name spark_mesos \
spark_mesos bash;
```
### 3. Verify running Spark on Mesos.
You can simply verify the processes of Spark and Mesos are running well in Docker with below command.
```
ps -ef
```
You can also check each application web UI for Mesos on `http://<hostname>:5050/cluster` and Spark on `http://<hostname>:8080/`.
### 4. Configure Spark interpreter in Zeppelin
```
export MASTER=mesos://127.0.1.1:5050
export MESOS_NATIVE_JAVA_LIBRARY=[PATH OF libmesos.so]
export SPARK_HOME=[PATH OF SPARK HOME]
```
Don't forget to set Spark `master` as `mesos://127.0.1.1:5050` in Zeppelin **Interpreters** setting page like below.
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png" />
### 5. Run Zeppelin with Spark interpreter
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:5050/#/frameworks` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" />

9
docs/install/upgrade.md
View file

@ -43,4 +43,11 @@ So, copying `notebook` and `conf` directory should be enough.
```
bin/zeppelin-daemon.sh start
```
```
## Migration Guide
### Upgrading from Zeppelin 0.6 to 0.7
- From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly.
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.

55
docs/rest-api/rest-notebook.md
View file

@ -450,12 +450,12 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</table>
<br/>
### Run a paragraph
### Run a paragraph asynchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
<td>This ```POST``` method runs the paragraph asynchronously by given notebook and paragraph id. This API always return SUCCESS even if the execution of the paragraph fails later because the API is asynchronous
</td>
</tr>
<tr>
@ -487,6 +487,56 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</table>
<br/>
### Run a paragraph synchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td> This ```POST``` method runs the paragraph synchronously by given notebook and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON input (optional, only needed when if you want to update dynamic form's value) </td>
<td><pre>
{
"name": "name of new notebook",
"params": {
"formLabel1": "value1",
"formLabel2": "value2"
}
}</pre></td>
</tr>
<tr>
<td> sample JSON response </td>
<td><pre>{"status": "OK"}</pre></td>
</tr>
<tr>
<td> sample JSON error </td>
<td><pre>
{
"status": "INTERNAL\_SERVER\_ERROR",
"body": {
"code": "ERROR",
"type": "TEXT",
"msg": "bash: -c: line 0: unexpected EOF while looking for matching ``'\nbash: -c: line 1: syntax error: unexpected end of file\nExitValue: 2"
}
}</pre></td>
</tr>
</table>
<br/>
### Stop a paragraph
<table class="table-configuration">
@ -922,4 +972,3 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</tr>
</table>

4
docs/screenshots.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Screenshots"
description: ""
title:
description:
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");

25
flink/pom.xml
View file

@ -120,21 +120,18 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -146,24 +143,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<!-- Scala Compiler -->
<plugin>
@ -319,7 +298,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
@ -339,10 +317,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>

2
ignite/pom.xml
View file

@ -116,8 +116,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>

13
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -15,7 +15,8 @@
package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
@ -418,11 +419,11 @@ public class JDBCInterpreter extends Interpreter {
} catch (Exception e) {
logger.error("Cannot run " + sql, e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getClass().toString()).append("\n");
stringBuilder.append(StringUtils.join(e.getStackTrace(), "\n"));
return new InterpreterResult(Code.ERROR, stringBuilder.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
String errorMsg = new String(baos.toByteArray(), StandardCharsets.UTF_8);
return new InterpreterResult(Code.ERROR, errorMsg);
}
}

255
notebook/r/note.json → notebook/2BWJFTXKJ/note.json
View file

File diff suppressed because one or more lines are too long

265
pom.xml
View file

@ -211,11 +211,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
@ -411,129 +406,6 @@
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>CONTRIBUTING.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>docs/CONTRIBUTING.md</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@ -816,6 +688,143 @@
</plugins>
</build>
</profile>
<profile>
<id>rat</id>
<activation>
<property><name>!skipRat</name></property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.rat-excludes/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>CONTRIBUTING.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>docs/CONTRIBUTING.md</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<exclude>**/horizontalbar_mockdata.txt</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<exclude>r/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

13
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -150,8 +150,17 @@ public class PythonInterpreter extends Interpreter {
* @return true if syntax error or exception has happened
*/
private boolean pythonErrorIn(String output) {
Matcher errorMatcher = errorInLastLine.matcher(output);
return errorMatcher.find();
boolean isError = false;
String[] outputMultiline = output.split("\n");
Matcher errorMatcher;
for (String row : outputMultiline) {
errorMatcher = errorInLastLine.matcher(row);
if (errorMatcher.find() == true) {
isError = true;
break;
}
}
return isError;
}
@Override

21
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -218,4 +218,25 @@ public class PythonInterpreterTest {
}
}
@Test
public void checkMultiRowErrorFails() {
PythonInterpreter pythonInterpreter = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties()
);
pythonInterpreter.open();
String codeRaiseException = "raise Exception(\"test exception\")";
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
assertNotNull("Interpreter result for raise exception is Null", ret);
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().length() > 0);
assertNotNull("Interpreter result for text is Null", ret);
String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
ret = pythonInterpreter.interpret(codePrintText, null);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().length() > 0);
}
}

8
r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala
View file

@ -111,10 +111,10 @@ object RInterpreter {
// These are the additional properties we need on top of the ones provided by the spark interpreters
lazy val props: Map[String, InterpreterProperty] = new InterpreterPropertyBuilder()
.add("rhadoop.cmd", SparkInterpreter.getSystemDefault("HADOOP_CMD", "rhadoop.cmd", ""), "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("HADOOP_STREAMING", "rhadooop.streamingjar", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", SparkInterpreter.getSystemDefault("RSCALA_DEBUG", "rscala.debug","false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", SparkInterpreter.getSystemDefault("RSCALA_TIMEOUT", "rscala.timeout","60"), "Timeout for rScala") // TEST: Implemented but not tested
.add("rhadoop.cmd", "HADOOP_CMD", "rhadoop.cmd", "", "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", "HADOOP_STREAMING", "rhadooop.streamingjar", "", "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", "RSCALA_DEBUG", "rscala.debug","false", "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", "RSCALA_TIMEOUT", "rscala.timeout","60", "Timeout for rScala") // TEST: Implemented but not tested
.build
def getProps() = {

63
scripts/docker/spark-cluster-managers/spark_mesos/Dockerfile Normal file
View file

@ -0,0 +1,63 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV HADOOP_PROFILE 2.3
ENV HADOOP_VERSION 2.3.0
# Update the image with the latest packages
RUN yum update -y; yum clean all
# Get utils
RUN yum install -y \
wget \
tar \
curl \
svn \
cyrus-sasl-md5 \
libevent2-devel \
&& \
yum clean all
# Remove old jdk
RUN yum remove java; yum remove jdk
# install jdk7
RUN yum install -y java-1.7.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java
ENV PATH $PATH:$JAVA_HOME/bin
# install spark
RUN curl -s http://www.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
# update boot script
COPY entrypoint.sh /etc/entrypoint.sh
RUN chown root.root /etc/entrypoint.sh
RUN chmod 700 /etc/entrypoint.sh
# install mesos
RUN wget http://repos.mesosphere.com/el/6/x86_64/RPMS/mesos-1.0.0-2.0.89.centos65.x86_64.rpm
RUN rpm -Uvh mesos-1.0.0-2.0.89.centos65.x86_64.rpm
#spark
EXPOSE 8080 7077 7072 8081 8082
#mesos
EXPOSE 5050 5051
ENTRYPOINT ["/etc/entrypoint.sh"]

48
scripts/docker/spark-cluster-managers/spark_mesos/entrypoint.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export SPARK_HOME=/usr/local/spark/
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=8080
export SPARK_WORKER_PORT=8888
export SPARK_WORKER_WEBUI_PORT=8081
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server/
# spark configuration
cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so" >> $SPARK_HOME/conf/spark-env.sh
cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf
echo "spark.master mesos://`hostname`:5050" >> $SPARK_HOME/conf/spark-defaults.conf
echo "spark.mesos.executor.home /usr/local/spark" >> $SPARK_HOME/conf/spark-defaults.conf
# run spark
cd $SPARK_HOME/sbin
./start-master.sh
./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
# start mesos
mesos-master --ip=0.0.0.0 --work_dir=/var/lib/mesos &> /var/log/mesos_master.log &
mesos-slave --master=0.0.0.0:5050 --work_dir=/var/lib/mesos --launcher=posix &> /var/log/mesos_slave.log &
CMD=${1:-"exit 0"}
if [[ "$CMD" == "-d" ]];
then
service sshd stop
/usr/sbin/sshd -D -d
else
/bin/bash -c "$*"
fi

20
spark-dependencies/pom.xml
View file

@ -959,26 +959,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

344
spark/pom.xml
View file

@ -301,27 +301,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/interpreter-setting.json</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
@ -482,27 +461,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.1</id>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.1.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<spark.version>1.1.1</spark.version>
<akka.version>2.2.3-shaded-protobuf</akka.version>
</properties>
</profile>
<profile>
<id>spark-1.2</id>
<dependencies>
@ -512,26 +470,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.2</id>
<properties>
<spark.version>1.2.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.2.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.3</id>
@ -544,27 +482,6 @@
</profile>
<profile>
<id>cassandra-spark-1.3</id>
<properties>
<spark.version>1.3.0</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.3.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.4</id>
<properties>
@ -575,27 +492,6 @@
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.4</id>
<properties>
<spark.version>1.4.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.4.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.5</id>
<properties>
@ -604,34 +500,6 @@
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
</properties>
<dependencies>
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.5</id>
<properties>
<spark.version>1.5.1</spark.version>
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
<guava.version>16.0.1</guava.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
@ -732,218 +600,6 @@
</properties>
</profile>
<profile>
<id>mapr3</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>1.0.3-mapr-3.0.3</hadoop.version>
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr40</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.4.1-mapr-1503</hadoop.version>
<yarn.version>2.4.1-mapr-1503</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr41</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.5.1-mapr-1503</hadoop.version>
<yarn.version>2.5.1-mapr-1503</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr50</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1506</hadoop.version>
<yarn.version>2.7.0-mapr-1506</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr51</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1602</hadoop.version>
<yarn.version>2.7.0-mapr-1602</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>yarn</id>
<properties>
<yarn.version>${hadoop.version}</yarn.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
</dependencies>
</profile>
<!-- include sparkr in the build -->
<profile>
<id>sparkr</id>

27
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -49,6 +49,7 @@ import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
@ -447,10 +448,11 @@ public class SparkInterpreter extends Interpreter {
}
private void setupConfForPySpark(SparkConf conf) {
String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null);
String pysparkBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue();
File pysparkPath;
if (null == pysparkBasePath) {
pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
pysparkBasePath =
new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue();
pysparkPath = new File(pysparkBasePath,
"interpreter" + File.separator + "spark" + File.separator + "pyspark");
} else {
@ -500,27 +502,6 @@ public class SparkInterpreter extends Interpreter {
return null != System.getenv("SPARK_SUBMIT");
}
public static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
return envValue;
}
}
if (propertyName != null && !propertyName.isEmpty()) {
String propValue = System.getProperty(propertyName);
if (propValue != null) {
return propValue;
}
}
return defaultValue;
}
public boolean printREPLOutput() {
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput"));
}

2
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -78,7 +78,7 @@ public class SparkRInterpreter extends Interpreter {
ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession());
}
ZeppelinRContext.setSqlContext(sparkInterpreter.getSQLContext());
ZeppelinRContext.setZepplinContext(sparkInterpreter.getZeppelinContext());
ZeppelinRContext.setZeppelinContext(sparkInterpreter.getZeppelinContext());
zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion);
try {

2
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java
View file

@ -33,7 +33,7 @@ public class ZeppelinRContext {
ZeppelinRContext.sparkContext = sparkContext;
}
public static void setZepplinContext(ZeppelinContext zeppelinContext) {
public static void setZeppelinContext(ZeppelinContext zeppelinContext) {
ZeppelinRContext.zeppelinContext = zeppelinContext;
}

20
zeppelin-display/pom.xml
View file

@ -81,6 +81,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -99,6 +100,7 @@
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
@ -106,24 +108,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>

10
zeppelin-examples/zeppelin-example-horizontalbar/pom.xml
View file

@ -105,16 +105,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/horizontalbar_mockdata.txt</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

18
zeppelin-interpreter/pom.xml
View file

@ -217,24 +217,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>*.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -345,10 +345,14 @@ public abstract class Interpreter {
}
public static void register(RegisteredInterpreter registeredInterpreter) {
// TODO(jongyoul): Error should occur when two same interpreter key with different settings
String interpreterKey = registeredInterpreter.getInterpreterKey();
if (!registeredInterpreters.containsKey(interpreterKey)) {
registeredInterpreters.put(interpreterKey, registeredInterpreter);
} else {
RegisteredInterpreter existInterpreter = registeredInterpreters.get(interpreterKey);
if (!existInterpreter.getProperties().equals(registeredInterpreter.getProperties())) {
logger.error("exist registeredInterpreter with the same key but has different settings.");
}
}
}

13
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
View file

@ -70,8 +70,15 @@ public class InterpreterProperty {
this.description = description;
}
public int hashCode() {
return this.toString().hashCode();
}
public boolean equals(Object o) {
return this.toString().equals(o.toString());
}
public String getValue() {
//TODO(jongyoul): Remove SparkInterpreter's getSystemDefault method
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
@ -90,7 +97,7 @@ public class InterpreterProperty {
@Override
public String toString() {
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s", envName,
propertyName, defaultValue, description);
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s}",
envName, propertyName, defaultValue, description);
}
}

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java
View file

@ -31,6 +31,13 @@ public class InterpreterPropertyBuilder {
return this;
}
public InterpreterPropertyBuilder add(String name, String envName, String propertyName,
String defaultValue, String description){
properties.put(name,
new InterpreterProperty(envName, propertyName, defaultValue, description));
return this;
}
public Map<String, InterpreterProperty> build(){
return properties;
}

48
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
/**
* This element stores the buffered
* append-data of paragraph's output.
*/
public class AppendOutputBuffer {
private String noteId;
private String paragraphId;
private String data;
public AppendOutputBuffer(String noteId, String paragraphId, String data) {
this.noteId = noteId;
this.paragraphId = paragraphId;
this.data = data;
}
public String getNoteId() {
return noteId;
}
public String getParagraphId() {
return paragraphId;
}
public String getData() {
return data;
}
}

118
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java Normal file
View file

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This thread sends paragraph's append-data
* periodically, rather than continously, with
* a period of BUFFER_TIME_MS. It handles append-data
* for all paragraphs across all notebooks.
*/
public class AppendOutputRunner implements Runnable {
private static final Logger logger =
LoggerFactory.getLogger(AppendOutputRunner.class);
public static final Long BUFFER_TIME_MS = new Long(100);
private static final Long SAFE_PROCESSING_TIME = new Long(10);
private static final Long SAFE_PROCESSING_STRING_SIZE = new Long(100000);
private final BlockingQueue<AppendOutputBuffer> queue = new LinkedBlockingQueue<>();
private final RemoteInterpreterProcessListener listener;
public AppendOutputRunner(RemoteInterpreterProcessListener listener) {
this.listener = listener;
}
@Override
public void run() {
Map<String, Map<String, StringBuilder> > noteMap = new HashMap<>();
List<AppendOutputBuffer> list = new LinkedList<>();
/* "drainTo" method does not wait for any element
* to be present in the queue, and thus this loop would
* continuosly run (with period of BUFFER_TIME_MS). "take()" method
* waits for the queue to become non-empty and then removes
* one element from it. Rest elements from queue (if present) are
* removed using "drainTo" method. Thus we save on some un-necessary
* cpu-cycles.
*/
try {
list.add(queue.take());
} catch (InterruptedException e) {
logger.error("Wait for OutputBuffer queue interrupted: " + e.getMessage());
}
Long processingStartTime = System.currentTimeMillis();
queue.drainTo(list);
for (AppendOutputBuffer buffer: list) {
String noteId = buffer.getNoteId();
String paragraphId = buffer.getParagraphId();
Map<String, StringBuilder> paragraphMap = (noteMap.containsKey(noteId)) ?
noteMap.get(noteId) : new HashMap<String, StringBuilder>();
StringBuilder builder = paragraphMap.containsKey(paragraphId) ?
paragraphMap.get(paragraphId) : new StringBuilder();
builder.append(buffer.getData());
paragraphMap.put(paragraphId, builder);
noteMap.put(noteId, paragraphMap);
}
Long processingTime = System.currentTimeMillis() - processingStartTime;
if (processingTime > SAFE_PROCESSING_TIME) {
logger.warn("Processing time for buffered append-output is high: " +
processingTime + " milliseconds.");
} else {
logger.debug("Processing time for append-output took "
+ processingTime + " milliseconds");
}
Long sizeProcessed = new Long(0);
for (String noteId: noteMap.keySet()) {
for (String paragraphId: noteMap.get(noteId).keySet()) {
String data = noteMap.get(noteId).get(paragraphId).toString();
sizeProcessed += data.length();
listener.onOutputAppend(noteId, paragraphId, data);
}
}
if (sizeProcessed > SAFE_PROCESSING_STRING_SIZE) {
logger.warn("Processing size for buffered append-output is high: " +
sizeProcessed + " characters.");
} else {
logger.debug("Processing size for append-output is " +
sizeProcessed + " characters");
}
}
public void appendBuffer(String noteId, String paragraphId, String outputToAppend) {
queue.offer(new AppendOutputBuffer(noteId, paragraphId, outputToAppend));
}
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -295,7 +295,9 @@ public class RemoteInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;

14
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -39,12 +39,18 @@ import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
/**
* Processes message from RemoteInterpreter process
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
private static final ScheduledExecutorService appendService =
Executors.newSingleThreadScheduledExecutor();
private final RemoteInterpreterProcessListener listener;
private final ApplicationEventListener appListener;
@ -72,6 +78,9 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
while (!shutdown) {
// wait and retry
@ -157,7 +166,7 @@ public class RemoteInterpreterEventPoller extends Thread {
String appId = outputAppend.get("appId");
if (appId == null) {
listener.onOutputAppend(noteId, paragraphId, outputToAppend);
runner.appendBuffer(noteId, paragraphId, outputToAppend);
} else {
appListener.onOutputAppend(noteId, paragraphId, appId, outputToAppend);
}
@ -192,6 +201,9 @@ public class RemoteInterpreterEventPoller extends Thread {
logger.error("Can't handle event " + event, e);
}
}
if (appendFuture != null) {
appendFuture.cancel(true);
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -34,6 +34,7 @@ import org.apache.zeppelin.display.*;
import org.apache.zeppelin.helium.*;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer;
import org.apache.zeppelin.interpreter.thrift.*;
import org.apache.zeppelin.resource.*;
import org.apache.zeppelin.scheduler.Job;
@ -133,7 +134,11 @@ public class RemoteInterpreterServer
public static void main(String[] args)
throws TTransportException, InterruptedException {
int port = Integer.parseInt(args[0]);
int port = ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT;
if (args.length > 0) {
port = Integer.parseInt(args[0]);
}
RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(port);
remoteInterpreterServer.start();
remoteInterpreterServer.join();
@ -280,7 +285,9 @@ public class RemoteInterpreterServer
@Override
public RemoteInterpreterResult interpret(String noteId, String className, String st,
RemoteInterpreterContext interpreterContext) throws TException {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
Interpreter intp = getInterpreter(noteId, className);
InterpreterContext context = convert(interpreterContext);

235
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java Normal file
View file

@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.After;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;
private static final int NUM_CLUBBED_EVENTS = 100;
private static final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
private static ScheduledFuture<?> future = null;
/* It is being accessed by multiple threads.
* While loop for 'loopForBufferCompletion' could
* run for-ever.
*/
private volatile static int numInvocations = 0;
@After
public void afterEach() {
if (future != null) {
future.cancel(true);
}
}
@Test
public void testSingleEvent() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String[][] buffer = {{"note", "para", "data\n"}};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend("note", "para", "data\n");
}
@Test
public void testMultipleEventsOfSameParagraph() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String para1 = "para1";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para1, "data2\n"},
{note1, para1, "data3\n"}
};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\ndata2\ndata3\n");
}
@Test
public void testMultipleEventsOfDifferentParagraphs() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String note2 = "note2";
String para1 = "para1";
String para2 = "para2";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para2, "data2\n"},
{note2, para1, "data3\n"},
{note2, para2, "data4\n"}
};
loopForCompletingEvents(listener, 4, buffer);
verify(listener, times(4)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\n");
verify(listener, times(1)).onOutputAppend(note1, para2, "data2\n");
verify(listener, times(1)).onOutputAppend(note2, para1, "data3\n");
verify(listener, times(1)).onOutputAppend(note2, para2, "data4\n");
}
@Test
public void testClubbedData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
Thread thread = new Thread(new BombardEvents(runner));
thread.start();
thread.join();
Thread.sleep(1000);
/* NUM_CLUBBED_EVENTS is a heuristic number.
* It has been observed that for 10,000 continuos event
* calls, 30-40 Web-socket calls are made. Keeping
* the unit-test to a pessimistic 100 web-socket calls.
*/
verify(listener, atMost(NUM_CLUBBED_EVENTS)).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
@Test
public void testWarnLoggerForLargeData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
String data = "data\n";
int numEvents = 100000;
for (int i=0; i<numEvents; i++) {
runner.appendBuffer("noteId", "paraId", data);
}
TestAppender appender = new TestAppender();
Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
Logger.getLogger(RemoteInterpreterEventPoller.class);
runner.run();
List<LoggingEvent> log;
int warnLogCounter;
LoggingEvent sizeWarnLogEntry = null;
do {
warnLogCounter = 0;
log = appender.getLog();
for (LoggingEvent logEntry: log) {
if (Level.WARN.equals(logEntry.getLevel())) {
sizeWarnLogEntry = logEntry;
warnLogCounter += 1;
}
}
} while(warnLogCounter != 2);
String loggerString = "Processing size for buffered append-output is high: " +
(data.length() * numEvents) + " characters.";
assertTrue(loggerString.equals(sizeWarnLogEntry.getMessage()));
}
private class BombardEvents implements Runnable {
private final AppendOutputRunner runner;
private BombardEvents(AppendOutputRunner runner) {
this.runner = runner;
}
@Override
public void run() {
String noteId = "noteId";
String paraId = "paraId";
for (int i=0; i<NUM_EVENTS; i++) {
runner.appendBuffer(noteId, paraId, "data\n");
}
}
}
private class TestAppender extends AppenderSkeleton {
private final List<LoggingEvent> log = new ArrayList<>();
@Override
public boolean requiresLayout() {
return false;
}
@Override
protected void append(final LoggingEvent loggingEvent) {
log.add(loggingEvent);
}
@Override
public void close() {
}
public List<LoggingEvent> getLog() {
return new ArrayList<>(log);
}
}
private void prepareInvocationCounts(RemoteInterpreterProcessListener listener) {
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
numInvocations += 1;
return null;
}
}).when(listener).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
private void loopForCompletingEvents(RemoteInterpreterProcessListener listener,
int numTimes, String[][] buffer) {
numInvocations = 0;
prepareInvocationCounts(listener);
AppendOutputRunner runner = new AppendOutputRunner(listener);
for (String[] bufferElement: buffer) {
runner.appendBuffer(bufferElement[0], bufferElement[1], bufferElement[2]);
}
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
long startTimeMs = System.currentTimeMillis();
while(numInvocations != numTimes) {
if (System.currentTimeMillis() - startTimeMs > 2000) {
fail("Buffered events were not sent for 2 seconds");
}
}
}
}

21
zeppelin-server/pom.xml
View file

@ -357,27 +357,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>src/test/java/com/webautomation/*</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>

90
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -319,7 +319,10 @@ public class NotebookRestApi {
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
String newNoteName = request.getName();
String newNoteName = null;
if (request != null) {
newNoteName = request.getName();
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
@ -567,7 +570,7 @@ public class NotebookRestApi {
}
/**
* Run paragraph job REST API
* Run asynchronously paragraph job REST API
*
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
@ -580,7 +583,7 @@ public class NotebookRestApi {
public Response runParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId, String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
LOG.info("run paragraph job asynchronously {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -593,22 +596,60 @@ public class NotebookRestApi {
}
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.setLastReplName(paragraph.getId());
note.persist(subject);
}
}
handleParagraphParams(message, note, paragraph);
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
/**
* Run synchronously a paragraph REST API
*
* @param noteId - noteId
* @param paragraphId - paragraphId
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("run/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraphSynchronously(@PathParam("notebookId") String noteId,
@PathParam("paragraphId") String paragraphId,
String message) throws
IOException, IllegalArgumentException {
LOG.info("run paragraph synchronously {} {} {}", noteId, paragraphId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
// handle params if presented
handleParagraphParams(message, note, paragraph);
if (paragraph.getListener() == null) {
note.initializeJobListenerForParagraph(paragraph);
}
paragraph.run();
final InterpreterResult result = paragraph.getResult();
if (result.code() == InterpreterResult.Code.SUCCESS) {
return new JsonResponse<>(Status.OK, result).build();
} else {
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, result).build();
}
}
/**
* Stop(delete) paragraph job REST API
*
@ -669,7 +710,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", request.getCronString());
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -697,7 +738,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -800,4 +841,21 @@ public class NotebookRestApi {
return new JsonResponse<>(Status.OK, notebooksFound).build();
}
private void handleParagraphParams(String message, Note note, Paragraph paragraph)
throws IOException {
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.setLastReplName(paragraph.getId());
note.persist(subject);
}
}
}
}

62
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -336,7 +336,7 @@ public class NotebookServer extends WebSocketServlet implements
List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.id(), m);
broadcast(note.getId(), m);
}
}
}
@ -473,11 +473,11 @@ public class NotebookServer extends WebSocketServlet implements
for (Note note : notes) {
Map<String, String> info = new HashMap<>();
if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) {
if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNotebookId)) {
continue;
}
info.put("id", note.id());
info.put("id", note.getId());
info.put("name", note.getName());
notesInfo.add(info);
}
@ -486,7 +486,7 @@ public class NotebookServer extends WebSocketServlet implements
}
public void broadcastNote(Note note) {
broadcast(note.id(), new Message(OP.NOTE).put("note", note));
broadcast(note.getId(), new Message(OP.NOTE).put("note", note));
}
public void broadcastInterpreterBindings(String noteId,
@ -544,7 +544,7 @@ public class NotebookServer extends WebSocketServlet implements
notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -568,7 +568,7 @@ public class NotebookServer extends WebSocketServlet implements
userAndRoles, notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -604,7 +604,7 @@ public class NotebookServer extends WebSocketServlet implements
note.setName(name);
note.setConfig(config);
if (cronUpdated) {
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
}
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
@ -643,7 +643,7 @@ public class NotebookServer extends WebSocketServlet implements
}
note.persist(subject);
addConnectionToNote(note.id(), (NotebookSocket) conn);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
broadcastNoteList(subject);
}
@ -697,7 +697,7 @@ public class NotebookServer extends WebSocketServlet implements
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
note.persist(subject);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
private void cloneNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -707,7 +707,7 @@ public class NotebookServer extends WebSocketServlet implements
String name = (String) fromMessage.get("name");
Note newNote = notebook.cloneNote(noteId, name, new AuthenticationInfo(fromMessage.principal));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.id(), (NotebookSocket) conn);
addConnectionToNote(newNote.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
broadcastNoteList(subject);
}
@ -810,12 +810,12 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(note.id()) == null) {
if (setting.getInterpreterGroup(note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(note.id()).getAngularObjectRegistry();
.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -852,17 +852,17 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(n.id()) == null) {
if (setting.getInterpreterGroup(n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(n.id()).getAngularObjectRegistry();
.getInterpreterGroup(n.getId()).getAngularObjectRegistry();
this.broadcastExcept(
n.id(),
n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", n.id())
.put("noteId", n.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -870,10 +870,10 @@ public class NotebookServer extends WebSocketServlet implements
}
} else { // broadcast to all web session for the note
this.broadcastExcept(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -1149,7 +1149,7 @@ public class NotebookServer extends WebSocketServlet implements
new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()),
ex);
p.setStatus(Status.ERROR);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
}
}
@ -1309,7 +1309,7 @@ public class NotebookServer extends WebSocketServlet implements
@Override
public void onProgressUpdate(Job job, int progress) {
notebookServer.broadcast(
note.id(),
note.getId(),
new Message(OP.PROGRESS).put("id", job.getId()).put("progress",
job.progress()));
}
@ -1384,15 +1384,15 @@ public class NotebookServer extends WebSocketServlet implements
}
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id())
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId())
.getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.id());
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(note.id()).getId())
.put("noteId", note.id())
intpSetting.getInterpreterGroup(note.getId()).getId())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId())
));
}
@ -1413,7 +1413,7 @@ public class NotebookServer extends WebSocketServlet implements
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) {
if (object.getNoteId() != null && !note.getId().equals(object.getNoteId())) {
continue;
}
@ -1424,11 +1424,11 @@ public class NotebookServer extends WebSocketServlet implements
}
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId()));
}
}
@ -1438,7 +1438,7 @@ public class NotebookServer extends WebSocketServlet implements
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (noteId != null && !note.id().equals(noteId)) {
if (noteId != null && !note.getId().equals(noteId)) {
continue;
}
@ -1446,7 +1446,7 @@ public class NotebookServer extends WebSocketServlet implements
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put(
"noteId", noteId).put("paragraphId", paragraphId));
}

2
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -134,7 +134,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
Note note = ZeppelinServer.notebook.createNote(null);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
assertThat(get, isAllowed());
get.addRequestHeader("Origin", "http://localhost");
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {

27
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NotebookAuthorization;
@ -144,6 +145,32 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(note1.getId(), null);
}
@Test
public void testCloneNotebook() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
PostMethod post = httpPost("/notebook/" + note1.getId(), "");
LOG.info("testCloneNotebook response\n" + post.getResponseBodyAsString());
assertThat(post, isCreated());
Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
String clonedNotebookId = (String) resp.get("body");
post.releaseConnection();
GetMethod get = httpGet("/notebook/" + clonedNotebookId);
assertThat(get, isAllowed());
Map<String, Object> resp2 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
assertEquals((String)resp2Body.get("name"), "Note " + clonedNotebookId);
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(clonedNotebookId, null);
}
}

14
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -79,7 +79,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55", p.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -91,7 +91,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
@ -118,7 +118,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[1] 3", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -141,7 +141,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -172,7 +172,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("10\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -204,7 +204,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p2.getStatus());
assertEquals("10", p2.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -216,7 +216,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isPyspark() && sparkVersionNumber >= 14) {
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {

9
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -325,6 +325,15 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
return;
}
if ($scope.newInterpreterSetting.name.indexOf('.') >= 0) {
BootstrapDialog.alert({
closable: true,
title: 'Add interpreter',
message: '\'.\' is invalid for interpreter name'
});
return;
}
if (_.findIndex($scope.interpreterSettings, {'name': $scope.newInterpreterSetting.name}) >= 0) {
BootstrapDialog.alert({
closable: true,

37
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -820,7 +820,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
var user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user;
var desc = 'Took ' + moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') +
'. Last updated by ' + user + ' at ' + moment(pdata.dateUpdated).format('MMMM DD YYYY, h:mm:ss A') + '.';
'. Last updated by ' + user + ' at ' + moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + '.';
if ($scope.isResultOutdated()) {
desc += ' (outdated)';
}
@ -865,6 +865,21 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
};
$scope.parseTableCell = function(cell) {
if (!isNaN(cell)) {
if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) {
return cell;
} else {
return Number(cell);
}
}
var d = moment(cell);
if (d.isValid()) {
return d;
}
return cell;
};
$scope.loadTableData = function(result) {
if (!result) {
return;
@ -898,8 +913,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
if (i === 0) {
columnNames.push({name: col, index: j, aggr: 'sum'});
} else {
cols.push(col);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: col});
var parsedCol = $scope.parseTableCell(col);
cols.push(parsedCol);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: parsedCol});
}
}
if (i !== 0) {
@ -978,7 +994,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
cells: function(row, col, prop) {
var cellProperties = {};
cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) {
if (!isNaN(value)) {
if (value instanceof moment) {
td.innerHTML = value._i;
} else if (!isNaN(value)) {
cellProperties.format = '0,0.[00000]';
td.style.textAlign = 'left';
Handsontable.renderers.NumericRenderer.apply(this, arguments);
@ -2575,7 +2593,16 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
});
$scope.$on('appendParagraphOutput', function(event, data) {
if ($scope.paragraph.id === data.paragraphId) {
/* It has been observed that append events
* can be errorneously called even if paragraph
* execution has ended, and in that case, no append
* should be made. Also, it was observed that between PENDING
* and RUNNING states, append-events can be called and we can't
* miss those, else during the length of paragraph run, few
* initial output line/s will be missing.
*/
if ($scope.paragraph.id === data.paragraphId &&
($scope.paragraph.status === 'RUNNING' || $scope.paragraph.status === 'PENDING')) {
if ($scope.flushStreamingOutput) {
$scope.clearTextOutput();
$scope.flushStreamingOutput = false;

6
zeppelin-zengine/pom.xml
View file

@ -128,12 +128,6 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>

30
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -272,10 +272,15 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getKeyStorePath() {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH)));
String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
public String getKeyStoreType() {
@ -297,10 +302,13 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getTrustStorePath() {
String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
if (path == null) {
return getKeyStorePath();
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(path);
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
@ -427,10 +435,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
public boolean getUseJdbcAlias() {
return getBoolean(ConfVars.ZEPPELIN_USE_JDBC_ALIAS);
}
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@ -557,9 +561,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true);
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000");
private String varName;
@SuppressWarnings("rawtypes")

3
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -506,6 +506,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
public InterpreterSetting createNewSetting(String name, String group,
List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
if (name.indexOf(".") >= 0) {
throw new IOException("'.' is invalid for InterpreterSetting name.");
}
InterpreterSetting setting = createFromInterpreterSettingRef(group);
setting.setName(name);
setting.setGroup(group);

86
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
@ -30,7 +32,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import com.google.gson.Gson;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,6 +57,11 @@ import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import static org.apache.commons.lang.StringUtils.EMPTY;
import static org.apache.commons.lang.StringUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
import static org.apache.commons.lang.StringUtils.isBlank;
/**
* Binded interpreters for a note
*/
@ -76,7 +82,7 @@ public class Note implements Serializable, ParagraphJobListener {
private String name = "";
private String id;
private AtomicReference<String> lastReplName = new AtomicReference<>(StringUtils.EMPTY);
private AtomicReference<String> lastReplName = new AtomicReference<>(EMPTY);
private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create();
private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
@ -122,7 +128,7 @@ public class Note implements Serializable, ParagraphJobListener {
private String getDefaultInterpreterName() {
InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
return null != setting ? setting.getName() : EMPTY;
}
void putDefaultReplName() {
@ -131,10 +137,6 @@ public class Note implements Serializable, ParagraphJobListener {
lastReplName.set(defaultInterpreterName);
}
public String id() {
return id;
}
public String getId() {
return id;
}
@ -172,6 +174,28 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (paragraphNote.getId().equals(this.getId())) {
throw new IllegalArgumentException(format("The paragraph %s from note %s " +
"does not belong to note %s", paragraph.getId(), paragraphNote.getId(),
this.getId()));
}
boolean foundParagraph = false;
for (Paragraph ownParagraph : paragraphs) {
if (paragraph.getId().equals(ownParagraph.getId())) {
paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
foundParagraph = true;
}
}
if (!foundParagraph) {
throw new IllegalArgumentException(format("Cannot find paragraph %s " +
"from note %s", paragraph.getId(), paragraphNote.getId()));
}
}
void setJobListenerFactory(JobListenerFactory jobListenerFactory) {
this.jobListenerFactory = jobListenerFactory;
}
@ -275,13 +299,17 @@ public class Note implements Serializable, ParagraphJobListener {
*/
private void addLastReplNameIfEmptyText(Paragraph p) {
String replName = lastReplName.get();
if (StringUtils.isEmpty(p.getText()) && StringUtils.isNotEmpty(replName)) {
if (isEmpty(p.getText()) && isNotEmpty(replName) && isBinding(replName)) {
p.setText(getInterpreterName(replName) + " ");
}
}
public boolean isBinding(String replName) {
return factory.getInterpreter(this.getId(), replName) != null;
}
private String getInterpreterName(String replName) {
return StringUtils.isBlank(replName) ? StringUtils.EMPTY : "%" + replName;
return isBlank(replName) ? EMPTY : "%" + replName;
}
/**
@ -292,7 +320,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph removeParagraph(String paragraphId) {
removeAllAngularObjectInParagraph(paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(id(), paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@ -354,8 +382,8 @@ public class Note implements Serializable, ParagraphJobListener {
if (index < 0 || index >= paragraphs.size()) {
if (throwWhenIndexIsOutOfBound) {
throw new IndexOutOfBoundsException("paragraph size is " + paragraphs.size() +
" , index is " + index);
throw new IndexOutOfBoundsException(
"paragraph size is " + paragraphs.size() + " , index is " + index);
} else {
return;
}
@ -428,7 +456,7 @@ public class Note implements Serializable, ParagraphJobListener {
return new HashMap<>();
}
}
private Map<String, String> populatePragraphInfo(Paragraph p) {
Map<String, String> info = new HashMap<>();
info.put("id", p.getId());
@ -476,27 +504,15 @@ public class Note implements Serializable, ParagraphJobListener {
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(getId(), requiredReplName);
if (intp == null) {
// TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter
if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) {
String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")");
logger.debug("New paragraph: {}", pText);
p.setEffectiveText(pText);
} else {
String intpExceptionMsg = String.format("%s",
p.getJobName()
+ "'s Interpreter "
+ requiredReplName + " not found"
);
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult = new InterpreterResult(
InterpreterResult.Code.ERROR, intpException.getMessage()
);
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
intp.getScheduler().submit(p);
@ -592,7 +608,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
private void setLastReplName(Paragraph lastParagraphStarted) {
if (StringUtils.isNotEmpty(lastParagraphStarted.getRequiredReplName())) {
if (isNotEmpty(lastParagraphStarted.getRequiredReplName())) {
lastReplName.set(lastParagraphStarted.getRequiredReplName());
}
}
@ -609,7 +625,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
void unpersist(AuthenticationInfo subject) throws IOException {
repo.remove(id(), subject);
repo.remove(getId(), subject);
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
View file

@ -36,7 +36,7 @@ public class NoteInfo {
}
public NoteInfo(Note note) {
id = note.id();
id = note.getId();
name = note.getName();
config = note.getConfig();
}

24
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -151,10 +151,10 @@ public class Notebook implements NoteEventListener {
Note note =
new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this);
synchronized (notes) {
notes.put(note.id(), note);
notes.put(note.getId(), note);
}
if (interpreterIds != null) {
bindInterpretersToNote(note.id(), interpreterIds);
bindInterpretersToNote(note.getId(), interpreterIds);
note.putDefaultReplName();
}
@ -239,10 +239,12 @@ public class Notebook implements NoteEventListener {
Note newNote = createNote(subject);
if (newNoteName != null) {
newNote.setName(newNoteName);
} else {
newNote.setName("Note " + newNote.getId());
}
// Copy the interpreter bindings
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.id());
bindInterpretersToNote(newNote.id(), boundInterpreterSettingsIds);
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);
List<Paragraph> paragraphs = sourceNote.getParagraphs();
for (Paragraph p : paragraphs) {
@ -419,15 +421,15 @@ public class Notebook implements NoteEventListener {
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.id(), note);
refreshCron(note.id());
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.id());
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
@ -508,11 +510,11 @@ public class Notebook implements NoteEventListener {
Collections.sort(noteList, new Comparator<Note>() {
@Override
public int compare(Note note1, Note note2) {
String name1 = note1.id();
String name1 = note1.getId();
if (note1.getName() != null) {
name1 = note1.getName();
}
String name2 = note2.id();
String name2 = note2.getId();
if (note2.getName() != null) {
name2 = note2.getName();
}
@ -593,14 +595,14 @@ public class Notebook implements NoteEventListener {
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.id());
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.id());
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )

22
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -53,7 +53,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private transient InterpreterFactory factory;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient String effectiveText;
String title;
String text;
@ -114,14 +113,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
this.dateUpdated = new Date();
}
public void setEffectiveText(String effectiveText) {
this.effectiveText = effectiveText;
}
public String getEffectiveText() {
return effectiveText;
}
public AuthenticationInfo getAuthenticationInfo() {
return authenticationInfo;
}
@ -153,7 +144,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getRequiredReplName() {
return getRequiredReplName(null != effectiveText ? effectiveText : text);
return getRequiredReplName(text);
}
public static String getRequiredReplName(String text) {
@ -182,7 +173,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getScriptBody() {
return getScriptBody(null != effectiveText ? effectiveText : text);
return getScriptBody(text);
}
public static String getScriptBody(String text) {
@ -359,7 +350,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
} finally {
InterpreterContext.remove();
effectiveText = null;
}
}
@ -451,13 +441,13 @@ public class Paragraph extends Job implements Serializable, Cloneable {
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(note.id()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.id()).getResourcePool();
registry = intpGroup.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
@ -470,7 +460,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
InterpreterContext interpreterContext = new InterpreterContext(
note.id(),
note.getId(),
getId(),
this.getTitle(),
this.getText(),

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -210,7 +210,7 @@ public class S3NotebookRepo implements NotebookRepo {
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json";
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -226,7 +226,7 @@ public class VFSNotebookRepo implements NotebookRepo {
FileObject rootDir = getRootDir();
FileObject noteDir = rootDir.resolveFile(note.id(), NameScope.CHILD);
FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD);
if (!noteDir.exists()) {
noteDir.createFolder();

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
View file

@ -176,7 +176,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
String notebook = GSON.toJson(note);
restApiClient.asyncPut(notebook);
LOG.info("ZeppelinHub REST API saving note {} ", note.id());
LOG.info("ZeppelinHub REST API saving note {} ", note.getId());
}
@Override

12
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -176,7 +176,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
factory.setInterpreters(note1.id(), factory.getDefaultInterpreterSettingList());
factory.setInterpreters(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -214,7 +214,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -231,7 +231,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
}
// when unbind interpreter
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// then
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
@ -255,7 +255,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// Unbind all interpreter from note
// NullPointerException shouldn't occur here
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// remove note
notebook.removeNote(note1.getId(), null);
@ -273,9 +273,9 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.id())) {
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
mock1IntpSettingId = setting.getId();
break;

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -157,4 +157,14 @@ public class InterpreterFactoryTest {
assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("note", "group1").getClassName());
}
@Test
public void testInvalidInterpreterSettingName() {
try {
factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
}
}
}

40
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
View file

@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@ -82,27 +83,6 @@ public class NoteTest {
assertEquals("Paragraph text", pText, pCaptor.getValue().getText());
}
@Test
public void runJdbcTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("mysql"))).thenReturn(null);
when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter);
when(interpreter.getScheduler()).thenReturn(scheduler);
String pText = "%mysql show databases";
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p = note.addParagraph();
p.setText(pText);
note.run(p.getId());
ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class);
verify(scheduler, only()).submit(pCaptor.capture());
verify(interpreterFactory, times(2)).getInterpreter(anyString(), anyString());
assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText());
assertEquals("Change paragraph text", pText, pCaptor.getValue().getText());
}
@Test
public void putDefaultReplNameIfInterpreterSettingAbsent() {
when(interpreterFactory.getDefaultInterpreterSetting(anyString()))
@ -138,6 +118,7 @@ public class NoteTest {
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName(); //set lastReplName
when(interpreterFactory.getInterpreter(note.getId(), "spark")).thenReturn(new MockInterpreter2(null));
Paragraph p = note.addParagraph();
@ -153,6 +134,7 @@ public class NoteTest {
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName(); //set lastReplName
when(interpreterFactory.getInterpreter(note.getId(), "spark")).thenReturn(new MockInterpreter2(null));
Paragraph p = note.insertParagraph(note.getParagraphs().size());
@ -171,4 +153,20 @@ public class NoteTest {
assertEquals("spark", note.getLastReplName());
}
@Test
public void isBindingTest() {
Note note = spy(new Note());
when(note.getId()).thenReturn("test1");
InterpreterFactory mockInterpreterFactory = mock(InterpreterFactory.class);
note.setInterpreterFactory(mockInterpreterFactory);
//when is not binding
assertFalse(note.isBinding("spark"));
//when is binding
when(mockInterpreterFactory.getInterpreter("test1", "spark")).
thenReturn(new MockInterpreter2(null));
assertTrue(note.isBinding("spark"));
}
}

82
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -171,7 +171,7 @@ public class NotebookTest implements JobListenerFactory{
notebook.reloadAllNotes(null);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 2);
assertEquals(notes.get(1).id(), copiedNote.id());
assertEquals(notes.get(1).getId(), copiedNote.getId());
assertEquals(notes.get(1).getName(), copiedNote.getName());
assertEquals(notes.get(1).getParagraphs(), copiedNote.getParagraphs());
@ -283,13 +283,13 @@ public class NotebookTest implements JobListenerFactory{
config.put("enabled", true);
config.put("cron", "* * * * * ?");
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
Thread.sleep(1*1000);
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
Thread.sleep(1000);
dateFinished = p.getDateFinished();
assertNotNull(dateFinished);
@ -318,7 +318,7 @@ public class NotebookTest implements JobListenerFactory{
config.put("cron", "1/3 * * * * ?");
config.put("releaseresource", true);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
@ -342,7 +342,7 @@ public class NotebookTest implements JobListenerFactory{
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
@ -398,6 +398,16 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(cp.getResult().message(), p.getResult().message());
}
@Test
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note cloneNote = notebook.cloneNote(note.getId(), null, null);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
}
@Test
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
@ -445,7 +455,7 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(1, ResourcePoolUtils.getAllResources().size());
// remove note
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
}
@ -463,20 +473,20 @@ public class NotebookTest implements JobListenerFactory{
Paragraph p1 = note.addParagraph();
// add paragraph scope object
registry.add("o1", "object1", note.id(), p1.getId());
registry.add("o1", "object1", note.getId(), p1.getId());
// add notebook scope object
registry.add("o2", "object2", note.id(), null);
registry.add("o2", "object2", note.getId(), null);
// add global scope object
registry.add("o3", "object3", null, null);
// remove notebook
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
// notebook scope or paragraph scope object should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o2", note.id(), p1.getId()));
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", note.getId(), p1.getId()));
// global object sould be remained
assertNotNull(registry.get("o3", null, null));
@ -496,10 +506,10 @@ public class NotebookTest implements JobListenerFactory{
Paragraph p1 = note.addParagraph();
// add paragraph scope object
registry.add("o1", "object1", note.id(), p1.getId());
registry.add("o1", "object1", note.getId(), p1.getId());
// add notebook scope object
registry.add("o2", "object2", note.id(), null);
registry.add("o2", "object2", note.getId(), null);
// add global scope object
registry.add("o3", "object3", null, null);
@ -508,10 +518,10 @@ public class NotebookTest implements JobListenerFactory{
note.removeParagraph(p1.getId());
// paragraph scope should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o1", note.getId(), null));
// notebook scope and global object sould be remained
assertNotNull(registry.get("o2", note.id(), null));
assertNotNull(registry.get("o2", note.getId(), null));
assertNotNull(registry.get("o3", null, null));
}
@ -527,7 +537,7 @@ public class NotebookTest implements JobListenerFactory{
.getAngularObjectRegistry();
// add local scope object
registry.add("o1", "object1", note.id(), null);
registry.add("o1", "object1", note.getId(), null);
// add global scope object
registry.add("o2", "object2", null, null);
@ -537,9 +547,9 @@ public class NotebookTest implements JobListenerFactory{
.getAngularObjectRegistry();
// local and global scope object should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
}
@Test
@ -548,43 +558,43 @@ public class NotebookTest implements JobListenerFactory{
Note note = notebook.createNote(null);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
// empty owners, readers and writers means note is public
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
notebookAuthorization.setOwners(note.id(),
notebookAuthorization.setOwners(note.getId(),
new HashSet<String>(Arrays.asList("user1")));
notebookAuthorization.setReaders(note.id(),
notebookAuthorization.setReaders(note.getId(),
new HashSet<String>(Arrays.asList("user1", "user2")));
notebookAuthorization.setWriters(note.id(),
notebookAuthorization.setWriters(note.getId(),
new HashSet<String>(Arrays.asList("user1")));
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), false);
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user1"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user3"))), false);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), false);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user1"))), true);
// Test clearing of permssions
notebookAuthorization.setReaders(note.id(), Sets.<String>newHashSet());
assertEquals(notebookAuthorization.isReader(note.id(),
notebookAuthorization.setReaders(note.getId(), Sets.<String>newHashSet());
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user3"))), true);
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
}
@Test
@ -777,8 +787,8 @@ public class NotebookTest implements JobListenerFactory{
note1.removeParagraph(p1.getId());
assertEquals(1, onParagraphRemove.get());
List<String> settings = notebook.getBindedInterpreterSettingsIds(note1.id());
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
List<String> settings = notebook.getBindedInterpreterSettingsIds(note1.getId());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
assertEquals(settings.size(), unbindInterpreter.get());
notebook.removeNote(note1.getId(), null);

26
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -73,32 +73,6 @@ public class ParagraphTest {
assertEquals("md", Paragraph.getRequiredReplName(text));
}
@Test
public void effectiveTextTest() {
InterpreterFactory interpreterFactory = mock(InterpreterFactory.class);
Interpreter interpreter = mock(Interpreter.class);
Note note = mock(Note.class);
Paragraph p = new Paragraph("paragraph", note, null, interpreterFactory);
p.setText("%h2 show databases");
p.setEffectiveText("%jdbc(h2) show databases");
assertEquals("Get right replName", "jdbc", p.getRequiredReplName());
assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody());
when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter);
when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE);
when(note.getId()).thenReturn("noteId");
try {
p.jobRun();
} catch (Throwable throwable) {
// Do nothing
}
assertEquals("Erase effective Text", "h2", p.getRequiredReplName());
assertEquals("Erase effective Text", "show databases", p.getScriptBody());
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given

2
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java
View file

@ -132,7 +132,7 @@ public class ZeppelinHubRepoTest {
public void testGetNote() throws IOException {
Note notebook = repo.get("AAAAA", null);
assertThat(notebook).isNotNull();
assertThat(notebook.id()).isEqualTo("2A94M5J1Z");
assertThat(notebook.getId()).isEqualTo("2A94M5J1Z");
}
@Test