Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix: Resolve merge conflict with 8f344db

Browse source
This commit is contained in:
1ambda 2016-09-21 00:23:27 +09:00
commit e6d41c8a92
106 changed files with 2237 additions and 1957 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/PULL_REQUEST_TEMPLATE vendored
View file

@ -1,6 +1,6 @@
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md
First time? Check out the contributing guide - https://zeppelin.apache.org/contribution/contributions.html
### What type of PR is it?

4
.gitignore vendored
View file

@ -47,6 +47,7 @@ zeppelin-web/bower_components
# R
/r/lib/
.Rhistory
/R/
# project level
/logs/
@ -108,3 +109,6 @@ tramp
# Generated by zeppelin-examples
/helium
# tmp files
/tmp/

26
.travis.yml
View file

@ -21,6 +21,7 @@ cache:
directories:
- .spark-dist
- ${HOME}/.m2/repository/.cache/maven-download-plugin
- .node_modules
addons:
apt:
@ -33,44 +34,49 @@ addons:
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.0.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.1
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- mkdir -p ~/R
- echo 'R_LIBS=~/R' > ~/.Renviron
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
@ -89,6 +95,7 @@ before_script:
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
- rm -rf .node_modules; cp -r zeppelin-web/node_modules .node_modules
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
@ -104,3 +111,4 @@ after_failure:
after_script:
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER

228
CONTRIBUTING.md
View file

@ -1,228 +0,0 @@
# How to contribute
**Zeppelin** is [Apache2 License](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md) Software.
Contributing to Zeppelin (Source code, Documents, Image, Website) means you agree to the Apache2 License.
1. Make sure your issue is not already in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
3. Contribute your patch via Pull Request.
Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it.
For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful.
## Creating a Pull Request
In order to make the review process easier, please follow this template when making a Pull Request:
```
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md
### What type of PR is it?
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
```
## Testing a Pull Request
You can also test and review a particular Pull Request. Here are two useful ways.
* Using a utility provided from Zeppelin.
```
dev/test_zeppelin_pr.py [# of PR]
```
For example, if you want to test `#513`, then the command will be:
```
dev/test_zeppelin_pr.py 513
```
* Another way is using [github/hub](https://github.com/github/hub).
```
hub checkout https://github.com/apache/zeppelin/pull/[# of PR]
```
The above two methods will help you test and review Pull Requests.
## Source Control Workflow
Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model.
## The Review Process
When a Pull Request is submitted, it is being merged or rejected by following review process.
* Anybody can be a reviewer and may comment on the change and suggest modifications.
* Reviewer can indicate that a patch looks suitable for merging with a comment such as: "Looks good", "LGTM", "+1".
* At least one indication of suitable for merging (e.g. "LGTM") from committer is required to be merged.
* Pull request is open for 1 or 2 days for potential additional review, unless it's got enough indication of suitable for merging.
* Committer can initiate lazy consensus ("Merge if there is no more discussion") and the code can be merged after certain time (normally 24 hours) when there is no review exists.
* Contributor can ping reviewers (including committer) by commenting 'Ready to review' or suitable indication.
## Becoming a Committer
The PPMC adds new committers from the active contributors, based on their contribution to Zeppelin. The qualifications for new committers include:
1. Sustained contributions: Committers should have a history of constant contributions to Zeppelin.
2. Quality of contributions: Committers more than any other community member should submit simple, well-tested, and well-designed patches.
3. Community involvement: Committers should have a constructive and friendly attitude in all community interactions. They should also be active on the dev, user list and reviewing patches. Also help new contributors and users.
## Setting up
Here are some things you will need to build and test Zeppelin.
### Software Configuration Management (SCM)
Zeppelin uses Git for its SCM system. `http://git.apache.org/zeppelin.git` you'll need git client installed in your development machine.
For write access, `https://git-wip-us.apache.org/repos/asf/zeppelin.git`
### Integrated Development Environment (IDE)
You are free to use whatever IDE you prefer, or your favorite command line editor.
### Project Structure
Zeppelin project is based on Maven. Maven works by convention & defines [directory structure] (https://maven.apache.org/guides/introduction/introduction-to-the-standard-directory-layout.html) for a project.
The top-level pom.xml describes the basic project structure. Currently Zeppelin has the following modules.
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>spark</module>
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>flink</module>
<module>ignite</module>
<module>lens</module>
<module>cassandra</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-distribution</module>
### Web Project Contribution Guidelines
If you plan on making a contribution to Zeppelin's WebApplication,
please check [its own contribution guidelines](https://github.com/apache/zeppelin/blob/master/zeppelin-web/CONTRIBUTING.md)
### Code convention
We are following Google Code style:
* [Java style](https://google.github.io/styleguide/javaguide.html)
* [Shell style](https://google.github.io/styleguide/shell.xml)
Check style report location are in `${submodule}/target/site/checkstyle.html`
Test coverage report location are in `${submodule}/target/site/cobertura/index.html`
#### Build Tools
To build the code, install
* Oracle Java 7
* Apache Maven
## Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git).
### git access
Get the source code on your development machine using git.
```
git clone git://git.apache.org/zeppelin.git zeppelin
```
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin
```
or with write access
```
git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git
```
### Fork repository
If you want not only build Zeppelin but also make change, then you need fork Zeppelin github mirror repository (https://github.com/apache/zeppelin) and make pull request.
## Build
```
mvn install
```
To skip test
```
mvn install -DskipTests
```
To build with specific spark / hadoop version
```
mvn install -Phadoop-2.2 -Dhadoop.version=2.2.0 -Pspark-1.3 -Dspark.version=1.3.0
```
## Tests
Each new File should have its own accompanying unit tests. Each new interpreter should have come with its tests.
Zeppelin has 3 types of tests:
1. Unit Tests: The unit tests run as part of each package's build. E.g. SparkInterpeter Module's unit test is SparkInterpreterTest
2. Integration Tests: The integration tests run after all modules are build. The integration tests launch an instance of Zeppelin server. ZeppelinRestApiTest is an example integration test.
3. GUI integration tests: These tests validate the Zeppelin UI elements. These tests require a running Zeppelin server and launches a web browser to validate Notebook UI elements like Notes and their execution. See ZeppelinIT as an example.
Currently the GUI integration tests are not run in the Maven and are only run in the CI environment when the pull request is submitted to github. Make sure to watch the [CI results] (https://travis-ci.org/apache/zeppelin/pull_requests) for your pull request.
## Continuous Integration
Zeppelin uses Travis for CI. In the project root there is .travis.yml that configures CI and [publishes CI results] (https://travis-ci.org/apache/zeppelin/builds)
## Run Zeppelin server in development mode
```
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
or use daemon script
```
bin/zeppelin-daemon start
```
Server will be run on http://localhost:8080
## JIRA
Zeppelin manages it's issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
## Where to Start
You can find issues for [beginner](https://issues.apache.org/jira/issues/?jql=(labels%20%3D%20newbie%20or%20labels%3D%20beginner)%20and%20project%3DZeppelin)
## Stay involved
Everyone is welcome to join our mailing list:
* [users@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for usage questions, help, and announcements [ [subscribe](mailto:users-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:users-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) ]
* [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for people who want to contribute code to Zeppelin.[ [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) ]
* [commits@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) is for commit messages and patches to Zeppelin. [ [subscribe](mailto:commits-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:commits-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) ]

10
README.md
View file

@ -3,7 +3,7 @@
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/zeppelin.png?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Contributing:** [Contribution Guide](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md)<br/>
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
**License:** [Apache 2.0](https://github.com/apache/zeppelin/blob/master/LICENSE)
@ -18,7 +18,7 @@ Core feature:
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org)
## Requirements
* Git
* Git
* Java 1.7
* Tested on Mac OSX, Ubuntu 14.X, CentOS 6.X, Windows 7 Pro SP1
* Maven (if you want to build from the source code)
@ -67,7 +67,7 @@ First of all, set your proxy configuration on Maven `settings.xml`.
</settings>
```
Then, run these commands from shell.
Then, run these commands from shell.
```
npm config set proxy http://localhost:3128
npm config set https-proxy http://localhost:3128
@ -87,10 +87,10 @@ git config --global --unset https.proxy
git config --global --unset url."http://".insteadOf
```
_Notes:_
_Notes:_
- If you are behind NTLM proxy you can use [Cntlm Authentication Proxy](http://cntlm.sourceforge.net/).
- Replace `localhost:3128` with the standard pattern `http://user:pwd@host:port`.
#### Install maven
```
wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz

7
bin/common.cmd
View file

@ -81,13 +81,6 @@ if not defined JAVA_OPTS (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_JAVA_OPTS (
set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_MEM (
set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM%
)
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%

9
bin/common.sh
View file

@ -121,15 +121,6 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
# jvm options for interpreter process
if [[ -z "${ZEPPELIN_INTP_JAVA_OPTS}" ]]; then
export ZEPPELIN_INTP_JAVA_OPTS="${ZEPPELIN_JAVA_OPTS}"
fi
if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
export ZEPPELIN_INTP_MEM="${ZEPPELIN_MEM}"
fi
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_INTP_OPTS

4
conf/zeppelin-env.cmd.template
View file

@ -20,8 +20,8 @@ REM set JAVA_HOME=
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options.
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.

4
conf/zeppelin-env.sh.template
View file

@ -20,8 +20,8 @@
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options.
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.

94
docs/CONTRIBUTING.md
View file

@ -1,94 +0,0 @@
# Contributing to Apache Zeppelin Documentation
## Folder Structure
`docs/` folder is organized as below:
```
docs/
├── _includes/themes/zeppelin
│ ├── _navigation.html
│ └── default.html
├── _layouts
├── _plugins
├── assets/themes/zeppelin -> {ASSET_PATH}
│ ├── bootstrap
│ ├── css
│ ├── img
│ └── js
├── development/ *.md
├── displaysystem/ *.md
├── install/ *.md
├── interpreter/ *.md
├── manual/ *.md
├── quickstart/ *.md
├── rest-api/ *.md
├── security/ *.md
├── storage/ *.md
├── Gemfile
├── Gemfile.lock
├── _config.yml
├── index.md
└── ...
```
- `_navigation.html`: the dropdown menu in navbar
- `default.html` & `_layouts/`: define default HTML layout
- `_plugins/`: custom plugin `*.rb` files can be placed in this folder. See [jekyll/plugins](https://jekyllrb.com/docs/plugins/) for the further information.
- `{ASSET_PATH}/css/style.css`: extra css components can be defined
- `{ASSET_PATH}/img/docs-img/`: image files used for document pages can be placed in this folder
- `{ASSET_PATH}/js/`: extra `.js` files can be placed
- `Gemfile`: defines bundle dependencies. They will be installed by `bundle install`.
- `Gemfile.lock`: when you run `bundle install`, bundler will persist all gems name and their version to this file. For the more details, see [Bundle "The Gemfile Lock"](http://bundler.io/v1.10/man/bundle-install.1.html#THE-GEMFILE-LOCK)
- `documentation_group`: `development/`, `displaysystem/`, `install/`, `interpreter/`...
- `_config.yml`: defines configuration options for docs website. See [jekyll/configuration](https://jekyllrb.com/docs/configuration/) for the other available config variables.
- `index.md`: the main page of `http://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>/`
## Markdown
Zeppelin documentation pages are written with [Markdown](http://daringfireball.net/projects/markdown/). It is possible to use [GitHub flavored syntax](https://help.github.com/categories/writing-on-github/) and intermix plain HTML.
## Front matter
Every page contains [YAML front matter](https://jekyllrb.com/docs/frontmatter/) block in their header. Don't forget to wrap the front matter list with triple-dashed lines(`---`) like below.
The document page should start this triple-dashed lines. Or you will face 404 error, since Jekyll can't find the page.
```
---
layout: page
title: "Apache Zeppelin Tutorial"
description: "This tutorial page contains a short walk-through tutorial that uses Apache Spark backend. Please note that this tutorial is valid for Spark 1.3 and higher."
group: quickstart
---
```
- `layout`: the default layout is `page` which is defined in `_layout/page.html`.
- `title`: the title for the document. Please note that if it needs to include `Zeppelin`, it should be `Apache Zeppelin`, not `Zeppelin`.
- `description`: a short description for the document. One or two sentences would be enough. This description also will be shown as an extract sentence when people search pages.
- `group`: a category of the document page
## Headings
All documents are structured with headings. From these headings, you can automatically generate a **Table of Contents**. There is a simple rule for Zeppelin docs headings.
```
# Level-1 heading <- used only for the main title
## Level-2 heading <- start with this
### Level-3 heading
#### Level-4 heading <- won't be converted in TOC from this level
```
## Table of contents(TOC)
```
<div id="toc"></div>
```
Add this line below `# main title` in order to generate a **Table of Contents**. Headings until `### (Level-3 heading)` are included to TOC.
Default setting options for TOC are definded in [here](https://github.com/apache/zeppelin/blob/master/docs/assets/themes/zeppelin/js/toc.js#L4).
## Adding new pages
If you're going to create new pages, there are some spots you need to add the location of the page.
- **Dropdown menu in navbar**: add your docs location to [_navigation.html](https://github.com/apache/zeppelin/blob/master/docs/_includes/themes/zeppelin/_navigation.html)
- **Main index**: add your docs below [What is the next?](http://zeppelin.apache.org/docs/latest/#what-is-the-next) section in [index.md](https://github.com/apache/zeppelin/blob/master/docs/index.md) with a short description. No need to do this if the page is for **Interpreters**.

8
docs/README.md
View file

@ -1,6 +1,6 @@
# Apache Zeppelin documentation
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest) is also generated from the files found in here.
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest/) is also generated from the files found in here.
## Build documentation
Zeppelin is using [Jekyll](https://jekyllrb.com/) which is a static site generator and [Github Pages](https://pages.github.com/) as a site publisher. For the more details, see [help.github.com/articles/about-github-pages-and-jekyll/](https://help.github.com/articles/about-github-pages-and-jekyll/).
@ -19,7 +19,7 @@ bundle install
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
On OS X 10.9, you may need to do
On OS X 10.9, you may need to do
```
xcode-select --install
@ -39,7 +39,7 @@ Using the above command, Jekyll will start a web server at `http://localhost:400
## Contribute to Zeppelin documentation
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://github.com/apache/zeppelin/blob/master/docs/CONTRIBUTING.md).
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://zeppelin.apache.org/contribution/contributions.html).
## For committers only
@ -49,7 +49,7 @@ If you wish to help us and contribute to Zeppelin Documentation, please look at
### Deploy to ASF svnpubsub infra
1. generate static website in `./_site`
```
# go to /docs under Zeppelin source
bundle exec jekyll build --safe

3
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -105,7 +105,8 @@
<li class="title"><span><b>Advanced</b><span></li>
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Contibute</b><span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>

3
docs/assets/themes/zeppelin/css/style.css
View file

@ -438,11 +438,10 @@ a.anchor {
.content table {
display: block;
width: 100%;
overflow: auto;
word-break: normal;
word-break: keep-all;
-webkit-overflow-scrolling: touch;
font-size: 90%;
font-size: 87%;
margin-top: 16px;
margin-bottom: 16px;
}

BIN
docs/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 76 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 125 KiB

6
docs/assets/themes/zeppelin/js/search.js
View file

@ -25,7 +25,7 @@ jQuery(function() {
this.field('group');
});
window.data = $.getJSON('/search_data.json');
window.data = $.getJSON('search_data.json');
window.data.then(function(loaded_data){
$.each(loaded_data, function(index, value){
window.idx.add(
@ -51,6 +51,8 @@ jQuery(function() {
function display_search_results(results) {
var $search_results = $("#search_results");
var zeppelin_version = {{site.ZEPPELIN_VERSION | jsonify}};
var base_url = {{site.JB.BASE_PATH | jsonify}};
var prod_url = {{site.production_url | jsonify}};
window.data.then(function(loaded_data) {
if (results.length) {
@ -59,7 +61,7 @@ jQuery(function() {
results.forEach(function(result) {
var item = loaded_data[result.ref];
var appendString = '<a href="'+item.url+'">'+item.title+'</a><div class="link">'+'https://zeppelin.apache.org/docs/'+zeppelin_version+item.url+'</div><p>'+item.excerpt+'</p><br/>';
var appendString = '<a href="'+base_url+item.url.trim()+'">'+item.title+'</a><div class="link">'+prod_url+base_url+item.url.trim()+'</div><p>'+item.excerpt+'</p><br/>';
$search_results.append(appendString);
});

7
docs/development/writingzeppelininterpreter.md
View file

@ -42,7 +42,7 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
```
{ZEPPELIN_INTERPRETER_DIR}/{YOUR_OWN_INTERPRETER_DIR}/interpreter-setting.json
@ -73,7 +73,7 @@ Here is an example of `interpreter-setting.json` on your own interpreter.
},
{
...
}
}
]
```
@ -207,7 +207,7 @@ Checkout some interpreters released with Zeppelin by default.
We welcome contribution to a new interpreter. Please follow these few steps:
- First, check out the general contribution guide [here](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md).
- First, check out the general contribution guide [here](https://zeppelin.apache.org/contribution/contributions.html).
- Follow the steps in [Make your own Interpreter](#make-your-own-interpreter) section above.
- Add your interpreter as in the [Configure your interpreter](#configure-your-interpreter) section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template).
- Add tests! They are run by [Travis](https://travis-ci.org/apache/zeppelin) for all changes and it is important that they are self-contained.
@ -215,4 +215,3 @@ We welcome contribution to a new interpreter. Please follow these few steps:
- Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`).
- Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE).
- Commit your changes and open a [Pull Request](https://github.com/apache/zeppelin/pulls) on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing.

8
docs/index.md
View file

@ -115,7 +115,7 @@ If you want to learn more about this feature, please visit [this page](./manual/
<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="./assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](./development/howtocontribute.html).
Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](https://zeppelin.apache.org/contribution/contributions.html).
Apache Zeppelin has a very active development community.
Join to our [Mailing list](https://zeppelin.apache.org/community.html) and report issues on [Jira Issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN).
@ -170,15 +170,15 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* Advanced
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-yarn-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-on-yarn-mode)
* [Zeppelin on Spark Cluster Mode (Mesos via Docker)](./install/spark_cluster_mode.html#spark-on-mesos-mode)
* Contribute
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
* [How to contribute (code)](./development/howtocontribute.html)
* [How to contribute (documentation website)](./development/howtocontributewebsite.html)
#### External Resources
#### External Resources
* [Mailing List](https://zeppelin.apache.org/community.html)
* [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
* [StackOverflow tag `apache-zeppelin`](http://stackoverflow.com/questions/tagged/apache-zeppelin)

8
docs/install/install.md
View file

@ -195,10 +195,10 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<table class="table-configuration">
<tr>
<th>zepplin-env.sh</th>
<th>zepplin-site.xml</th>
<th>zeppelin-env.sh</th>
<th>zeppelin-site.xml</th>
<th>Default value</th>
<th>Description</th>
<th class="col-md-4">Description</th>
</tr>
<tr>
<td>ZEPPELIN_PORT</td>
@ -354,7 +354,7 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=&lt;accountName&gt;;AccountKey=&lt;accountKey&gt;</code></td>
<td>The Azure storage account connection string<br />i.e. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=&lt;accountName&gt;;<br/>AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>

66
docs/install/spark_cluster_mode.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Apache Zeppelin on Spark cluster mode"
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using docker scripts."
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager(Standalone, Hadoop Yarn, Apache Mesos) with Apache Zeppelin using docker scripts."
group: install
---
<!--
@ -113,7 +113,7 @@ docker run -it \
### 3. Verify running Spark on YARN.
You can simply verify the processes of Spark and YARN is running well in Docker with below command.
You can simply verify the processes of Spark and YARN are running well in Docker with below command.
```
ps -ef
@ -140,3 +140,65 @@ Don't forget to set Spark `master` as `yarn-client` in Zeppelin **Interpreters**
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/yarn_applications.png" />
## Spark on Mesos mode
You can simply set up [Spark on Mesos](http://spark.apache.org/docs/latest/running-on-mesos.html) docker environment with below steps.
### 1. Build Docker file
```
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_mesos
docker build -t "spark_mesos" .
```
### 2. Run docker
```
docker run --net=host -it \
-p 8080:8080 \
-p 7077:7077 \
-p 8888:8888 \
-p 8081:8081 \
-p 8082:8082 \
-p 5050:5050 \
-p 5051:5051 \
-p 4040:4040 \
-h sparkmaster \
--name spark_mesos \
spark_mesos bash;
```
### 3. Verify running Spark on Mesos.
You can simply verify the processes of Spark and Mesos are running well in Docker with below command.
```
ps -ef
```
You can also check each application web UI for Mesos on `http://<hostname>:5050/cluster` and Spark on `http://<hostname>:8080/`.
### 4. Configure Spark interpreter in Zeppelin
```
export MASTER=mesos://127.0.1.1:5050
export MESOS_NATIVE_JAVA_LIBRARY=[PATH OF libmesos.so]
export SPARK_HOME=[PATH OF SPARK HOME]
```
Don't forget to set Spark `master` as `mesos://127.0.1.1:5050` in Zeppelin **Interpreters** setting page like below.
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png" />
### 5. Run Zeppelin with Spark interpreter
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:5050/#/frameworks` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" />

9
docs/install/upgrade.md
View file

@ -43,4 +43,11 @@ So, copying `notebook` and `conf` directory should be enough.
```
bin/zeppelin-daemon.sh start
```
```
## Migration Guide
### Upgrading from Zeppelin 0.6 to 0.7
- From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly.
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.

2
docs/interpreter/elasticsearch.md
View file

@ -243,7 +243,7 @@ delete /index/type/id
```
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```bash
%elasticsearch

2
docs/interpreter/hive.md
View file

@ -151,7 +151,7 @@ select * from my_table;
You can also run multiple queries up to 10 by default. Changing these settings is not implemented yet.
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```sql
%hive

2
docs/interpreter/livy.md
View file

@ -174,7 +174,7 @@ When Zeppelin server is running with authentication enabled, then this interpret
## Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features.
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features.
```
%livy.pyspark

11
docs/interpreter/markdown.md
View file

@ -31,7 +31,7 @@ In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to inv
In Zeppelin, Markdown interpreter is enabled by default and uses the [markdown4j](https://github.com/jdcasey/markdown4j) parser.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="60%" />
<img src="../assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="60%" />
## Configuration
<table class="table-configuration">
@ -51,17 +51,16 @@ In Zeppelin, Markdown interpreter is enabled by default and uses the [markdown4j
The following example demonstrates the basic usage of Markdown in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />
<img src="../assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />
### Markdown4j Parser
`markdown4j` parser provides [YUML](http://yuml.me/) and [Websequence](https://www.websequencediagrams.com/) extensions
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example-markdown4j-parser.png" width="70%" />
<img src="../assets/themes/zeppelin/img/docs-img/markdown-example-markdown4j-parser.png" width="70%" />
### Pegdown Parser
`pegdown` parser provides github flavored markdown.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example-pegdown-parser.png" width="70%" />
`pegdown` parser provides github flavored markdown.
<img src="../assets/themes/zeppelin/img/docs-img/markdown-example-pegdown-parser.png" width="70%" />

4
docs/interpreter/python.md
View file

@ -108,7 +108,7 @@ z.show(plt, height='150px', fmt='svg')
## Pandas integration
Apache Zeppelin [Table Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table) provides built-in data visualization capabilities. Python interpreter leverages it to visualize Pandas DataFrames though similar `z.show()` API, same as with [Matplotlib integration](#matplotlib-integration).
Apache Zeppelin [Table Display System](../displaysystem/basicdisplaysystem.html#table) provides built-in data visualization capabilities. Python interpreter leverages it to visualize Pandas DataFrames though similar `z.show()` API, same as with [Matplotlib integration](#matplotlib-integration).
Example:
@ -120,7 +120,7 @@ z.show(rates)
## SQL over Pandas DataFrames
There is a convenience `%python.sql` interpreter that matches Apache Spark experience in Zeppelin and enables usage of SQL language to query [Pandas DataFrames](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html) and visualization of results though built-in [Table Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table).
There is a convenience `%python.sql` interpreter that matches Apache Spark experience in Zeppelin and enables usage of SQL language to query [Pandas DataFrames](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html) and visualization of results though built-in [Table Display System](../displaysystem/basicdisplaysystem.html#table).
**Pre-requests**

2
docs/interpreter/shell.md
View file

@ -63,6 +63,6 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
## Example
The following example demonstrates the basic usage of Shell in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/shell-example.png" />
<img src="../assets/themes/zeppelin/img/docs-img/shell-example.png" />
If you need further information about **Zeppelin Interpreter Setting** for using Shell interpreter, please read [What is interpreter setting?](../manual/interpreters.html#what-is-interpreter-setting) section first.

105
docs/interpreter/spark.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Apache Spark Interpreter for Apache Zeppelin"
description: "Apache Spark is a fast and general-purpose cluster computing system. It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs."
description: "Apache Spark is a fast and general-purpose cluster computing system. It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution engine."
group: interpreter
---
<!--
@ -25,9 +25,8 @@ limitations under the License.
## Overview
[Apache Spark](http://spark.apache.org) is a fast and general-purpose cluster computing system.
It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs
Apache Spark is supported in Zeppelin with
Spark Interpreter group, which consists of five interpreters.
It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs.
Apache Spark is supported in Zeppelin with Spark interpreter group which consists of below five interpreters.
<table class="table-configuration">
<tr>
@ -38,25 +37,25 @@ Spark Interpreter group, which consists of five interpreters.
<tr>
<td>%spark</td>
<td>SparkInterpreter</td>
<td>Creates a SparkContext and provides a scala environment</td>
<td>Creates a SparkContext and provides a Scala environment</td>
</tr>
<tr>
<td>%pyspark</td>
<td>%spark.pyspark</td>
<td>PySparkInterpreter</td>
<td>Provides a python environment</td>
<td>Provides a Python environment</td>
</tr>
<tr>
<td>%r</td>
<td>%spark.r</td>
<td>SparkRInterpreter</td>
<td>Provides an R environment with SparkR support</td>
</tr>
<tr>
<td>%sql</td>
<td>%spark.sql</td>
<td>SparkSQLInterpreter</td>
<td>Provides a SQL environment</td>
</tr>
<tr>
<td>%dep</td>
<td>%spark.dep</td>
<td>DepInterpreter</td>
<td>Dependency loader</td>
</tr>
@ -139,111 +138,113 @@ You can also set other Spark properties which are not listed in the table. For a
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
### 1. Export SPARK_HOME
In **conf/zeppelin-env.sh**, export `SPARK_HOME` environment variable with your Spark installation path.
In `conf/zeppelin-env.sh`, export `SPARK_HOME` environment variable with your Spark installation path.
for example
For example,
```bash
export SPARK_HOME=/usr/lib/spark
```
You can optionally export HADOOP\_CONF\_DIR and SPARK\_SUBMIT\_OPTIONS
You can optionally export `HADOOP_CONF_DIR` and `SPARK_SUBMIT_OPTIONS`
```bash
export HADOOP_CONF_DIR=/usr/lib/hadoop
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"
```
For Windows, ensure you have `winutils.exe` in `%HADOOP_HOME%\bin`. For more details please see [Problems running Hadoop on Windows](https://wiki.apache.org/hadoop/WindowsProblems)
For Windows, ensure you have `winutils.exe` in `%HADOOP_HOME%\bin`. Please see [Problems running Hadoop on Windows](https://wiki.apache.org/hadoop/WindowsProblems) for the details.
### 2. Set master in Interpreter menu
After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type.
for example,
For example,
* **local[*]** in local mode
* **spark://master:7077** in standalone cluster
* **yarn-client** in Yarn client mode
* **mesos://host:5050** in Mesos cluster
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. (Zeppelin 0.5.6-incubating release works up to Spark 1.6.1 )
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way.
For the further information about Spark & Zeppelin version compatibility, please refer to "Available Interpreters" section in [Zeppelin download page](https://zeppelin.apache.org/download.html).
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
## SparkContext, SQLContext, ZeppelinContext
SparkContext, SQLContext, ZeppelinContext are automatically created and exposed as variable names 'sc', 'sqlContext' and 'z', respectively, both in scala and python environments.
## SparkContext, SQLContext, SparkSession, ZeppelinContext
SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.
Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x.
> Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
> Note that Scala/Python/R environment shares the same SparkContext, SQLContext and ZeppelinContext instance.
<a name="dependencyloading"> </a>
## Dependency Management
There are two ways to load external library in spark interpreter. First is using Interpreter setting menu and second is loading Spark properties.
There are two ways to load external libraries in Spark interpreter. First is using interpreter setting menu and second is loading Spark properties.
### 1. Setting Dependencies via Interpreter Setting
Please see [Dependency Management](../manual/dependencymanagement.html) for the details.
### 2. Loading Spark Properties
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations.
The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in `conf/zeppelin-env.sh`. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properties that user can set to distribute libraries are:
<table class="table-configuration">
<tr>
<th>spark-defaults.conf</th>
<th>SPARK_SUBMIT_OPTIONS</th>
<th>Applicable Interpreter</th>
<th>Description</th>
</tr>
<tr>
<td>spark.jars</td>
<td>--jars</td>
<td>%spark</td>
<td>Comma-separated list of local jars to include on the driver and executor classpaths.</td>
</tr>
<tr>
<td>spark.jars.packages</td>
<td>--packages</td>
<td>%spark</td>
<td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.</td>
<td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be <code>groupId:artifactId:version</code>.</td>
</tr>
<tr>
<td>spark.files</td>
<td>--files</td>
<td>%pyspark</td>
<td>Comma-separated list of files to be placed in the working directory of each executor.</td>
</tr>
</table>
> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
Here are few examples:
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
* `SPARK_SUBMIT_OPTIONS` in `conf/zeppelin-env.sh`
```bash
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
```
* `SPARK_HOME/conf/spark-defaults.conf`
* SPARK_HOME/conf/spark-defaults.conf
```
spark.jars /path/mylib1.jar,/path/mylib2.jar
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
```
### 3. Dynamic Dependency Loading via %dep interpreter
> Note: `%dep` interpreter is deprecated since v0.6.0.
`%dep` interpreter load libraries to `%spark` and `%pyspark` but not to `%spark.sql` interpreter so we recommend you to use first option instead.
### 3. Dynamic Dependency Loading via %spark.dep interpreter
> Note: `%spark.dep` interpreter is deprecated since v0.6.0.
`%spark.dep` interpreter loads libraries to `%spark` and `%spark.pyspark` but not to `%spark.sql` interpreter. So we recommend you to use the first option instead.
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%spark.dep` interpreter.
* Load libraries recursively from Maven repository
* Load libraries recursively from maven repository
* Load libraries from local filesystem
* Add additional maven repository
* Automatically add libraries to SparkCluster (You can turn off)
Dep interpreter leverages scala environment. So you can write any Scala code here.
Note that `%dep` interpreter should be used before `%spark`, `%pyspark`, `%sql`.
Dep interpreter leverages Scala environment. So you can write any Scala code here.
Note that `%spark.dep` interpreter should be used before `%spark`, `%spark.pyspark`, `%spark.sql`.
Here's usages.
```scala
%dep
%spark.dep
z.reset() // clean up previously added artifact and repository
// add maven repository
@ -277,11 +278,11 @@ z.load("groupId:artifactId:version").local()
```
## ZeppelinContext
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
Zeppelin automatically injects `ZeppelinContext` as variable `z` in your Scala/Python environment. `ZeppelinContext` provides some additional functions and utilities.
### Object Exchange
ZeppelinContext extends map and it's shared between scala, python environment.
So you can put some object from scala and read it from python, vise versa.
`ZeppelinContext` extends map and it's shared between Scala and Python environment.
So you can put some objects from Scala and read it from Python, vice versa.
<div class="codetabs">
<div data-lang="scala" markdown="1">
@ -298,7 +299,7 @@ z.put("objName", myObject)
{% highlight python %}
# Get object from python
%pyspark
%spark.pyspark
myObject = z.get("objName")
{% endhighlight %}
@ -307,8 +308,8 @@ myObject = z.get("objName")
### Form Creation
ZeppelinContext provides functions for creating forms.
In scala and python environments, you can create forms programmatically.
`ZeppelinContext` provides functions for creating forms.
In Scala and Python environments, you can create forms programmatically.
<div class="codetabs">
<div data-lang="scala" markdown="1">
@ -333,7 +334,7 @@ z.select("formName", "option1", Seq(("option1", "option1DisplayName"),
<div data-lang="python" markdown="1">
{% highlight python %}
%pyspark
%spark.pyspark
# Create text input form
z.input("formName")
@ -354,8 +355,8 @@ z.select("formName", [("option1", "option1DisplayName"),
In sql environment, you can create form in simple template.
```
%sql
```sql
%spark.sql
select * from ${table=defaultTableName} where text like '%${search}%'
```
@ -364,7 +365,7 @@ To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.
## Interpreter setting option
Interpreter setting can choose one of 'shared', 'scoped', 'isolated' option. Spark interpreter creates separate scala compiler per each notebook but share a single SparkContext in 'scoped' mode (experimental). It creates separate SparkContext per each notebook in 'isolated' mode.
You can choose one of `shared`, `scoped` and `isolated` options wheh you configure Spark interpreter. Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). It creates separated SparkContext per each notebook in `isolated` mode.
## Setting up Zeppelin with Kerberos
@ -377,14 +378,14 @@ Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark o
1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf.
This is to make the server communicate with KDC.
2. Set SPARK\_HOME in `[ZEPPELIN\_HOME]/conf/zeppelin-env.sh` to use spark-submit
(Additionally, you might have to set `export HADOOP\_CONF\_DIR=/etc/hadoop/conf`)
2. Set `SPARK_HOME` in `[ZEPPELIN_HOME]/conf/zeppelin-env.sh` to use spark-submit
(Additionally, you might have to set `export HADOOP_CONF_DIR=/etc/hadoop/conf`)
3. Add the two properties below to spark configuration (`[SPARK_HOME]/conf/spark-defaults.conf`):
3. Add the two properties below to Spark configuration (`[SPARK_HOME]/conf/spark-defaults.conf`):
spark.yarn.principal
spark.yarn.keytab
> **NOTE:** If you do not have access to the above spark-defaults.conf file, optionally, you may add the lines to the Spark Interpreter through the Interpreter tab in the Zeppelin UI.
> **NOTE:** If you do not have permission to access for the above spark-defaults.conf file, optionally, you can add the above lines to the Spark Interpreter setting through the Interpreter tab in the Zeppelin UI.
4. That's it. Play with Zeppelin!

12
docs/manual/dependencymanagement.md
View file

@ -33,8 +33,8 @@ When your code requires external library, instead of doing download/copy/restart
<hr>
<div class="row">
<div class="col-md-6">
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png" />
<a data-lightbox="compiler" href="../assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png">
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png" />
</a>
</div>
<div class="col-md-6" style="padding-top:30px">
@ -52,11 +52,11 @@ When your code requires external library, instead of doing download/copy/restart
<hr>
<div class="row">
<div class="col-md-6">
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png" />
<a data-lightbox="compiler" href="../assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png">
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png" />
</a>
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png" />
<a data-lightbox="compiler" href="../assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png">
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png" />
</a>
</div>
<div class="col-md-6" style="padding-top:30px">

55
docs/rest-api/rest-notebook.md
View file

@ -450,12 +450,12 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</table>
<br/>
### Run a paragraph
### Run a paragraph asynchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
<td>This ```POST``` method runs the paragraph asynchronously by given notebook and paragraph id. This API always return SUCCESS even if the execution of the paragraph fails later because the API is asynchronous
</td>
</tr>
<tr>
@ -487,6 +487,56 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</table>
<br/>
### Run a paragraph synchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td> This ```POST``` method runs the paragraph synchronously by given notebook and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON input (optional, only needed when if you want to update dynamic form's value) </td>
<td><pre>
{
"name": "name of new notebook",
"params": {
"formLabel1": "value1",
"formLabel2": "value2"
}
}</pre></td>
</tr>
<tr>
<td> sample JSON response </td>
<td><pre>{"status": "OK"}</pre></td>
</tr>
<tr>
<td> sample JSON error </td>
<td><pre>
{
"status": "INTERNAL\_SERVER\_ERROR",
"body": {
"code": "ERROR",
"type": "TEXT",
"msg": "bash: -c: line 0: unexpected EOF while looking for matching ``'\nbash: -c: line 1: syntax error: unexpected end of file\nExitValue: 2"
}
}</pre></td>
</tr>
</table>
<br/>
### Stop a paragraph
<table class="table-configuration">
@ -922,4 +972,3 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</tr>
</table>

4
docs/screenshots.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Screenshots"
description: ""
title:
description:
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");

27
flink/pom.xml
View file

@ -34,7 +34,7 @@
<description>Zeppelin flink support</description>
<properties>
<flink.version>1.0.3</flink.version>
<flink.version>1.1.2</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<scala.macros.version>2.0.1</scala.macros.version>
</properties>
@ -120,21 +120,18 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -146,24 +143,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<!-- Scala Compiler -->
<plugin>
@ -319,7 +298,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
@ -339,10 +317,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>

38
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -30,6 +30,7 @@ import java.util.*;
import org.apache.flink.api.scala.FlinkILoop;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
@ -42,6 +43,7 @@ import org.slf4j.LoggerFactory;
import scala.Console;
import scala.None;
import scala.Option;
import scala.Some;
import scala.collection.JavaConversions;
import scala.collection.immutable.Nil;
@ -83,14 +85,25 @@ public class FlinkInterpreter extends Interpreter {
startFlinkMiniCluster();
}
flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
flinkIloop = new FlinkILoop(getHost(),
getPort(),
flinkConf,
(BufferedReader) null,
new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
org.apache.flink.api.scala.ExecutionEnvironment env = flinkIloop.scalaEnv();
env.getConfig().disableSysoutLogging();
org.apache.flink.api.scala.ExecutionEnvironment benv =
flinkIloop.scalaBenv();
//new ExecutionEnvironment(remoteBenv)
org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv =
flinkIloop.scalaSenv();
senv.getConfig().disableSysoutLogging();
benv.getConfig().disableSysoutLogging();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
@ -100,13 +113,19 @@ public class FlinkInterpreter extends Interpreter {
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
binder.put("env", env);
imain.interpret("val env = _binder.get(\"env\").asInstanceOf["
+ env.getClass().getName() + "]");
binder.put("benv", benv);
imain.interpret("val benv = _binder.get(\"benv\").asInstanceOf["
+ benv.getClass().getName() + "]");
binder.put("senv", senv);
imain.interpret("val senv = _binder.get(\"senv\").asInstanceOf["
+ senv.getClass().getName() + "]");
}
private boolean localMode() {
@ -313,8 +332,6 @@ public class FlinkInterpreter extends Interpreter {
}
}
@Override
public void cancel(InterpreterContext context) {
}
@ -354,4 +371,5 @@ public class FlinkInterpreter extends Interpreter {
static final String toString(Object o) {
return (o instanceof String) ? (String) o : "";
}
}

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -81,7 +81,7 @@ public class FlinkInterpreterTest {
@Test
public void testWordCount() {
flink.interpret("val text = env.fromElements(\"To be or not to be\")", context);
flink.interpret("val text = benv.fromElements(\"To be or not to be\")", context);
flink.interpret("val counts = text.flatMap { _.toLowerCase.split(\" \") }.map { (_, 1) }.groupBy(0).sum(1)", context);
InterpreterResult result = flink.interpret("counts.print()", context);
assertEquals(Code.SUCCESS, result.code());

2
ignite/pom.xml
View file

@ -116,8 +116,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>

96
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -15,7 +15,8 @@
package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
@ -26,7 +27,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -167,7 +168,7 @@ public class JDBCInterpreter extends Interpreter {
logger.debug("propertiesMap: {}", propertiesMap);
if (!StringUtils.isAnyEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
if (!StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
JDBCSecurityImpl.createSecureConfiguration(property);
}
for (String propertyKey : propertiesMap.keySet()) {
@ -214,49 +215,52 @@ public class JDBCInterpreter extends Interpreter {
Class.forName(properties.getProperty(DRIVER_KEY));
final String url = properties.getProperty(URL_KEY);
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = DriverManager.getConnection(url, properties);
} else {
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = DriverManager.getConnection(url + ";hive.server2.proxy.user=" + user,
properties);
if (StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
connection = DriverManager.getConnection(url, properties);
} else {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = DriverManager.getConnection(url, properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return DriverManager.getConnection(url, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = DriverManager.getConnection(url + ";hive.server2.proxy.user=" + user,
properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return DriverManager.getConnection(url, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
}
}
}
break;
break;
default:
connection = DriverManager.getConnection(url, properties);
default:
connection = DriverManager.getConnection(url, properties);
}
}
}
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
return connection;
@ -418,11 +422,11 @@ public class JDBCInterpreter extends Interpreter {
} catch (Exception e) {
logger.error("Cannot run " + sql, e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getClass().toString()).append("\n");
stringBuilder.append(StringUtils.join(e.getStackTrace(), "\n"));
return new InterpreterResult(Code.ERROR, stringBuilder.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
String errorMsg = new String(baos.toByteArray(), StandardCharsets.UTF_8);
return new InterpreterResult(Code.ERROR, errorMsg);
}
}

1
jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java
View file

@ -18,7 +18,6 @@ package org.apache.zeppelin.jdbc.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.jdbc.SqlCompleter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

255
notebook/r/note.json → notebook/2BWJFTXKJ/note.json
View file

File diff suppressed because one or more lines are too long

275
pom.xml
View file

@ -211,11 +211,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
@ -389,7 +384,7 @@
</plugin>
<!--TODO(alex): make part of the build and reconcile conflicts
<plugin>
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.4</version>
@ -398,12 +393,12 @@
<id>default</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict>
</configuration>
</plugin>
-->
@ -411,129 +406,6 @@
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>CONTRIBUTING.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>docs/CONTRIBUTING.md</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@ -816,6 +688,141 @@
</plugins>
</build>
</profile>
<profile>
<id>rat</id>
<activation>
<property><name>!skipRat</name></property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.rat-excludes/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<exclude>**/horizontalbar_mockdata.txt</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<exclude>r/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

13
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -150,8 +150,17 @@ public class PythonInterpreter extends Interpreter {
* @return true if syntax error or exception has happened
*/
private boolean pythonErrorIn(String output) {
Matcher errorMatcher = errorInLastLine.matcher(output);
return errorMatcher.find();
boolean isError = false;
String[] outputMultiline = output.split("\n");
Matcher errorMatcher;
for (String row : outputMultiline) {
errorMatcher = errorInLastLine.matcher(row);
if (errorMatcher.find() == true) {
isError = true;
break;
}
}
return isError;
}
@Override

21
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -218,4 +218,25 @@ public class PythonInterpreterTest {
}
}
@Test
public void checkMultiRowErrorFails() {
PythonInterpreter pythonInterpreter = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties()
);
pythonInterpreter.open();
String codeRaiseException = "raise Exception(\"test exception\")";
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
assertNotNull("Interpreter result for raise exception is Null", ret);
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().length() > 0);
assertNotNull("Interpreter result for text is Null", ret);
String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
ret = pythonInterpreter.interpret(codePrintText, null);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().length() > 0);
}
}

8
r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala
View file

@ -111,10 +111,10 @@ object RInterpreter {
// These are the additional properties we need on top of the ones provided by the spark interpreters
lazy val props: Map[String, InterpreterProperty] = new InterpreterPropertyBuilder()
.add("rhadoop.cmd", SparkInterpreter.getSystemDefault("HADOOP_CMD", "rhadoop.cmd", ""), "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("HADOOP_STREAMING", "rhadooop.streamingjar", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", SparkInterpreter.getSystemDefault("RSCALA_DEBUG", "rscala.debug","false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", SparkInterpreter.getSystemDefault("RSCALA_TIMEOUT", "rscala.timeout","60"), "Timeout for rScala") // TEST: Implemented but not tested
.add("rhadoop.cmd", "HADOOP_CMD", "rhadoop.cmd", "", "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", "HADOOP_STREAMING", "rhadooop.streamingjar", "", "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", "RSCALA_DEBUG", "rscala.debug","false", "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", "RSCALA_TIMEOUT", "rscala.timeout","60", "Timeout for rScala") // TEST: Implemented but not tested
.build
def getProps() = {

63
scripts/docker/spark-cluster-managers/spark_mesos/Dockerfile Normal file
View file

@ -0,0 +1,63 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV HADOOP_PROFILE 2.3
ENV HADOOP_VERSION 2.3.0
# Update the image with the latest packages
RUN yum update -y; yum clean all
# Get utils
RUN yum install -y \
wget \
tar \
curl \
svn \
cyrus-sasl-md5 \
libevent2-devel \
&& \
yum clean all
# Remove old jdk
RUN yum remove java; yum remove jdk
# install jdk7
RUN yum install -y java-1.7.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java
ENV PATH $PATH:$JAVA_HOME/bin
# install spark
RUN curl -s http://www.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
# update boot script
COPY entrypoint.sh /etc/entrypoint.sh
RUN chown root.root /etc/entrypoint.sh
RUN chmod 700 /etc/entrypoint.sh
# install mesos
RUN wget http://repos.mesosphere.com/el/6/x86_64/RPMS/mesos-1.0.0-2.0.89.centos65.x86_64.rpm
RUN rpm -Uvh mesos-1.0.0-2.0.89.centos65.x86_64.rpm
#spark
EXPOSE 8080 7077 7072 8081 8082
#mesos
EXPOSE 5050 5051
ENTRYPOINT ["/etc/entrypoint.sh"]

48
scripts/docker/spark-cluster-managers/spark_mesos/entrypoint.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export SPARK_HOME=/usr/local/spark/
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=8080
export SPARK_WORKER_PORT=8888
export SPARK_WORKER_WEBUI_PORT=8081
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server/
# spark configuration
cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so" >> $SPARK_HOME/conf/spark-env.sh
cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf
echo "spark.master mesos://`hostname`:5050" >> $SPARK_HOME/conf/spark-defaults.conf
echo "spark.mesos.executor.home /usr/local/spark" >> $SPARK_HOME/conf/spark-defaults.conf
# run spark
cd $SPARK_HOME/sbin
./start-master.sh
./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
# start mesos
mesos-master --ip=0.0.0.0 --work_dir=/var/lib/mesos &> /var/log/mesos_master.log &
mesos-slave --master=0.0.0.0:5050 --work_dir=/var/lib/mesos --launcher=posix &> /var/log/mesos_slave.log &
CMD=${1:-"exit 0"}
if [[ "$CMD" == "-d" ]];
then
service sshd stop
/usr/sbin/sshd -D -d
else
/bin/bash -c "$*"
fi

20
spark-dependencies/pom.xml
View file

@ -959,26 +959,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

344
spark/pom.xml
View file

@ -301,27 +301,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/interpreter-setting.json</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
@ -482,27 +461,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.1</id>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.1.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<spark.version>1.1.1</spark.version>
<akka.version>2.2.3-shaded-protobuf</akka.version>
</properties>
</profile>
<profile>
<id>spark-1.2</id>
<dependencies>
@ -512,26 +470,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.2</id>
<properties>
<spark.version>1.2.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.2.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.3</id>
@ -544,27 +482,6 @@
</profile>
<profile>
<id>cassandra-spark-1.3</id>
<properties>
<spark.version>1.3.0</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.3.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.4</id>
<properties>
@ -575,27 +492,6 @@
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.4</id>
<properties>
<spark.version>1.4.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.4.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.5</id>
<properties>
@ -604,34 +500,6 @@
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
</properties>
<dependencies>
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.5</id>
<properties>
<spark.version>1.5.1</spark.version>
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
<guava.version>16.0.1</guava.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
@ -732,218 +600,6 @@
</properties>
</profile>
<profile>
<id>mapr3</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>1.0.3-mapr-3.0.3</hadoop.version>
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr40</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.4.1-mapr-1503</hadoop.version>
<yarn.version>2.4.1-mapr-1503</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr41</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.5.1-mapr-1503</hadoop.version>
<yarn.version>2.5.1-mapr-1503</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr50</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1506</hadoop.version>
<yarn.version>2.7.0-mapr-1506</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr51</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1602</hadoop.version>
<yarn.version>2.7.0-mapr-1602</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>yarn</id>
<properties>
<yarn.version>${hadoop.version}</yarn.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
</dependencies>
</profile>
<!-- include sparkr in the build -->
<profile>
<id>sparkr</id>

58
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -49,6 +49,7 @@ import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
@ -328,6 +329,7 @@ public class SparkInterpreter extends Interpreter {
}
setupConfForPySpark(conf);
setupConfForSparkR(conf);
Class SparkSession = Utils.findClass("org.apache.spark.sql.SparkSession");
Object builder = Utils.invokeStaticMethod(SparkSession, "builder");
Utils.invokeMethod(builder, "config", new Class[]{ SparkConf.class }, new Object[]{ conf });
@ -442,15 +444,17 @@ public class SparkInterpreter extends Interpreter {
}
}
setupConfForPySpark(conf);
setupConfForSparkR(conf);
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
}
private void setupConfForPySpark(SparkConf conf) {
String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null);
String pysparkBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue();
File pysparkPath;
if (null == pysparkBasePath) {
pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
pysparkBasePath =
new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue();
pysparkPath = new File(pysparkBasePath,
"interpreter" + File.separator + "spark" + File.separator + "pyspark");
} else {
@ -492,6 +496,35 @@ public class SparkInterpreter extends Interpreter {
}
}
private void setupConfForSparkR(SparkConf conf) {
String sparkRBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue();
File sparkRPath;
if (null == sparkRBasePath) {
sparkRBasePath =
new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue();
sparkRPath = new File(sparkRBasePath,
"interpreter" + File.separator + "spark" + File.separator + "R");
} else {
sparkRPath = new File(sparkRBasePath, "R" + File.separator + "lib");
}
sparkRPath = new File(sparkRPath, "sparkr.zip");
if (sparkRPath.exists() && sparkRPath.isFile()) {
String archives = null;
if (conf.contains("spark.yarn.dist.archives")) {
archives = conf.get("spark.yarn.dist.archives");
}
if (archives != null) {
archives = archives + "," + sparkRPath + "#sparkr";
} else {
archives = sparkRPath + "#sparkr";
}
conf.set("spark.yarn.dist.archives", archives);
} else {
logger.warn("sparkr.zip is not found, sparkr may not work.");
}
}
static final String toString(Object o) {
return (o instanceof String) ? (String) o : "";
}
@ -500,27 +533,6 @@ public class SparkInterpreter extends Interpreter {
return null != System.getenv("SPARK_SUBMIT");
}
public static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
return envValue;
}
}
if (propertyName != null && !propertyName.isEmpty()) {
String propValue = System.getProperty(propertyName);
if (propValue != null) {
return propValue;
}
}
return defaultValue;
}
public boolean printREPLOutput() {
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput"));
}

2
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -78,7 +78,7 @@ public class SparkRInterpreter extends Interpreter {
ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession());
}
ZeppelinRContext.setSqlContext(sparkInterpreter.getSQLContext());
ZeppelinRContext.setZepplinContext(sparkInterpreter.getZeppelinContext());
ZeppelinRContext.setZeppelinContext(sparkInterpreter.getZeppelinContext());
zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion);
try {

10
spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
View file

@ -52,13 +52,19 @@ public class SparkVersion {
if (pos > 0) {
numberPart = versionString.substring(0, pos);
}
version = Integer.parseInt(numberPart.replaceAll("\\.", ""));
String versions[] = numberPart.split("\\.");
int major = Integer.parseInt(versions[0]);
int minor = Integer.parseInt(versions[1]);
int patch = Integer.parseInt(versions[2]);
// version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch));
} catch (Exception e) {
logger.error("Can not recognize Spark version " + versionString +
". Assume it's a future release", e);
// assume it is future release
version = 999;
version = 99999;
}
}

2
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java
View file

@ -33,7 +33,7 @@ public class ZeppelinRContext {
ZeppelinRContext.sparkContext = sparkContext;
}
public static void setZepplinContext(ZeppelinContext zeppelinContext) {
public static void setZeppelinContext(ZeppelinContext zeppelinContext) {
ZeppelinRContext.zeppelinContext = zeppelinContext;
}

2
spark/src/main/resources/R/zeppelin_sparkr.R
View file

@ -42,7 +42,7 @@ assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv)
# setup spark env
assign(".sc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkContext"), envir = SparkR:::.sparkREnv)
assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
if (version >= 200) {
if (version >= 20000) {
assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv)
assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
}

6
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -107,9 +107,9 @@ class PyZeppelinContext(dict):
class SparkVersion(object):
SPARK_1_4_0 = 140
SPARK_1_3_0 = 130
SPARK_2_0_0 = 200
SPARK_1_4_0 = 10400
SPARK_1_3_0 = 10300
SPARK_2_0_0 = 20000
def __init__(self, versionNumber):
self.version = versionNumber

9
spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
View file

@ -24,7 +24,7 @@ public class SparkVersionTest {
@Test
public void testUnknownSparkVersion() {
assertEquals(999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
assertEquals(99999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
}
@Test
@ -33,6 +33,8 @@ public class SparkVersionTest {
assertFalse(SparkVersion.fromVersionString("1.5.9").isUnsupportedVersion());
assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion());
assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion());
// should support spark2 version of HDP 2.5
assertFalse(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion());
}
@Test
@ -40,6 +42,9 @@ public class SparkVersionTest {
// test equals
assertEquals(SparkVersion.SPARK_1_2_0, SparkVersion.fromVersionString("1.2.0"));
assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
// test spark2 version of HDP 2.5
assertEquals(SparkVersion.SPARK_2_0_0, SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245"));
// test newer than
assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_2_0));
@ -60,7 +65,7 @@ public class SparkVersionTest {
assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0));
// conversion
assertEquals(120, SparkVersion.SPARK_1_2_0.toNumber());
assertEquals(10200, SparkVersion.SPARK_1_2_0.toNumber());
assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString());
}
}

20
zeppelin-display/pom.xml
View file

@ -81,6 +81,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -99,6 +100,7 @@
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
@ -106,24 +108,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>

10
zeppelin-examples/zeppelin-example-horizontalbar/pom.xml
View file

@ -105,16 +105,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/horizontalbar_mockdata.txt</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

18
zeppelin-interpreter/pom.xml
View file

@ -217,24 +217,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>*.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -338,17 +338,21 @@ public abstract class Interpreter {
@Deprecated
public static void register(String name, String group, String className,
boolean defaultInterpreter, Map<String, InterpreterProperty> properties) {
logger.error("Static initialization is deprecated. You should change it to use " +
"interpreter-setting.json in your jar or " +
"interpreter/{interpreter}/interpreter-setting.json");
logger.warn("Static initialization is deprecated for interpreter {}, You should change it " +
"to use interpreter-setting.json in your jar or " +
"interpreter/{interpreter}/interpreter-setting.json", name);
register(new RegisteredInterpreter(name, group, className, defaultInterpreter, properties));
}
public static void register(RegisteredInterpreter registeredInterpreter) {
// TODO(jongyoul): Error should occur when two same interpreter key with different settings
String interpreterKey = registeredInterpreter.getInterpreterKey();
if (!registeredInterpreters.containsKey(interpreterKey)) {
registeredInterpreters.put(interpreterKey, registeredInterpreter);
} else {
RegisteredInterpreter existInterpreter = registeredInterpreters.get(interpreterKey);
if (!existInterpreter.getProperties().equals(registeredInterpreter.getProperties())) {
logger.error("exist registeredInterpreter with the same key but has different settings.");
}
}
}

14
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
View file

@ -70,8 +70,16 @@ public class InterpreterProperty {
this.description = description;
}
public int hashCode() {
return this.toString().hashCode();
}
public boolean equals(Object o) {
if (o == null) return false;
return this.toString().equals(o.toString());
}
public String getValue() {
//TODO(jongyoul): Remove SparkInterpreter's getSystemDefault method
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
@ -90,7 +98,7 @@ public class InterpreterProperty {
@Override
public String toString() {
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s", envName,
propertyName, defaultValue, description);
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s}",
envName, propertyName, defaultValue, description);
}
}

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java
View file

@ -31,6 +31,13 @@ public class InterpreterPropertyBuilder {
return this;
}
public InterpreterPropertyBuilder add(String name, String envName, String propertyName,
String defaultValue, String description){
properties.put(name,
new InterpreterProperty(envName, propertyName, defaultValue, description));
return this;
}
public Map<String, InterpreterProperty> build(){
return properties;
}

48
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
/**
* This element stores the buffered
* append-data of paragraph's output.
*/
public class AppendOutputBuffer {
private String noteId;
private String paragraphId;
private String data;
public AppendOutputBuffer(String noteId, String paragraphId, String data) {
this.noteId = noteId;
this.paragraphId = paragraphId;
this.data = data;
}
public String getNoteId() {
return noteId;
}
public String getParagraphId() {
return paragraphId;
}
public String getData() {
return data;
}
}

118
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java Normal file
View file

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This thread sends paragraph's append-data
* periodically, rather than continously, with
* a period of BUFFER_TIME_MS. It handles append-data
* for all paragraphs across all notebooks.
*/
public class AppendOutputRunner implements Runnable {
private static final Logger logger =
LoggerFactory.getLogger(AppendOutputRunner.class);
public static final Long BUFFER_TIME_MS = new Long(100);
private static final Long SAFE_PROCESSING_TIME = new Long(10);
private static final Long SAFE_PROCESSING_STRING_SIZE = new Long(100000);
private final BlockingQueue<AppendOutputBuffer> queue = new LinkedBlockingQueue<>();
private final RemoteInterpreterProcessListener listener;
public AppendOutputRunner(RemoteInterpreterProcessListener listener) {
this.listener = listener;
}
@Override
public void run() {
Map<String, Map<String, StringBuilder> > noteMap = new HashMap<>();
List<AppendOutputBuffer> list = new LinkedList<>();
/* "drainTo" method does not wait for any element
* to be present in the queue, and thus this loop would
* continuosly run (with period of BUFFER_TIME_MS). "take()" method
* waits for the queue to become non-empty and then removes
* one element from it. Rest elements from queue (if present) are
* removed using "drainTo" method. Thus we save on some un-necessary
* cpu-cycles.
*/
try {
list.add(queue.take());
} catch (InterruptedException e) {
logger.error("Wait for OutputBuffer queue interrupted: " + e.getMessage());
}
Long processingStartTime = System.currentTimeMillis();
queue.drainTo(list);
for (AppendOutputBuffer buffer: list) {
String noteId = buffer.getNoteId();
String paragraphId = buffer.getParagraphId();
Map<String, StringBuilder> paragraphMap = (noteMap.containsKey(noteId)) ?
noteMap.get(noteId) : new HashMap<String, StringBuilder>();
StringBuilder builder = paragraphMap.containsKey(paragraphId) ?
paragraphMap.get(paragraphId) : new StringBuilder();
builder.append(buffer.getData());
paragraphMap.put(paragraphId, builder);
noteMap.put(noteId, paragraphMap);
}
Long processingTime = System.currentTimeMillis() - processingStartTime;
if (processingTime > SAFE_PROCESSING_TIME) {
logger.warn("Processing time for buffered append-output is high: " +
processingTime + " milliseconds.");
} else {
logger.debug("Processing time for append-output took "
+ processingTime + " milliseconds");
}
Long sizeProcessed = new Long(0);
for (String noteId: noteMap.keySet()) {
for (String paragraphId: noteMap.get(noteId).keySet()) {
String data = noteMap.get(noteId).get(paragraphId).toString();
sizeProcessed += data.length();
listener.onOutputAppend(noteId, paragraphId, data);
}
}
if (sizeProcessed > SAFE_PROCESSING_STRING_SIZE) {
logger.warn("Processing size for buffered append-output is high: " +
sizeProcessed + " characters.");
} else {
logger.debug("Processing size for append-output is " +
sizeProcessed + " characters");
}
}
public void appendBuffer(String noteId, String paragraphId, String outputToAppend) {
queue.offer(new AppendOutputBuffer(noteId, paragraphId, outputToAppend));
}
}

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -295,7 +295,9 @@ public class RemoteInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
@ -525,4 +527,11 @@ public class RemoteInterpreter extends Interpreter {
public void setEnv(Map<String, String> env) {
this.env = env;
}
public void addEnv(Map<String, String> env) {
if (this.env == null) {
this.env = new HashMap<>();
}
this.env.putAll(env);
}
}

14
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -39,12 +39,18 @@ import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
/**
* Processes message from RemoteInterpreter process
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
private static final ScheduledExecutorService appendService =
Executors.newSingleThreadScheduledExecutor();
private final RemoteInterpreterProcessListener listener;
private final ApplicationEventListener appListener;
@ -72,6 +78,9 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
while (!shutdown) {
// wait and retry
@ -157,7 +166,7 @@ public class RemoteInterpreterEventPoller extends Thread {
String appId = outputAppend.get("appId");
if (appId == null) {
listener.onOutputAppend(noteId, paragraphId, outputToAppend);
runner.appendBuffer(noteId, paragraphId, outputToAppend);
} else {
appListener.onOutputAppend(noteId, paragraphId, appId, outputToAppend);
}
@ -192,6 +201,9 @@ public class RemoteInterpreterEventPoller extends Thread {
logger.error("Can't handle event " + event, e);
}
}
if (appendFuture != null) {
appendFuture.cancel(true);
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -34,6 +34,7 @@ import org.apache.zeppelin.display.*;
import org.apache.zeppelin.helium.*;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer;
import org.apache.zeppelin.interpreter.thrift.*;
import org.apache.zeppelin.resource.*;
import org.apache.zeppelin.scheduler.Job;
@ -133,7 +134,11 @@ public class RemoteInterpreterServer
public static void main(String[] args)
throws TTransportException, InterruptedException {
int port = Integer.parseInt(args[0]);
int port = ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT;
if (args.length > 0) {
port = Integer.parseInt(args[0]);
}
RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(port);
remoteInterpreterServer.start();
remoteInterpreterServer.join();
@ -280,7 +285,9 @@ public class RemoteInterpreterServer
@Override
public RemoteInterpreterResult interpret(String noteId, String className, String st,
RemoteInterpreterContext interpreterContext) throws TException {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
Interpreter intp = getInterpreter(noteId, className);
InterpreterContext context = convert(interpreterContext);

158
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterResultTest.java
View file

@ -23,91 +23,89 @@ import static org.junit.Assert.fail;
import org.junit.Test;
public class InterpreterResultTest {
@Test
public void testTextType() {
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"this is a TEXT type");
assertEquals("No magic",InterpreterResult.Type.TEXT, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%this is a TEXT type");
assertEquals("No magic",InterpreterResult.Type.TEXT, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%\n");
assertEquals("No magic",InterpreterResult.Type.TEXT, result.type());
}
@Test
public void testSimpleMagicType() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table col1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table\ncol1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before magic word %table col1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
}
@Test
public void testTextType() {
public void testComplexMagicType() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before %table col1\tcol2\naaa\t123\n");
assertEquals("some text before magic return magic",InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%html <h3> This is a hack </h3> %table\n col1\tcol2\naaa\t123\n");
assertEquals("magic A before magic B return magic A",InterpreterResult.Type.HTML, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before magic word %table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3>");
assertEquals("text & magic A before magic B return magic A" ,InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3> %table col1\naaa\n123\n");
assertEquals("magic A, magic B, magic a' return magic A" ,InterpreterResult.Type.TABLE, result.type());
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "this is a TEXT type");
assertEquals("No magic", InterpreterResult.Type.TEXT, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%this is a TEXT type");
assertEquals("No magic", InterpreterResult.Type.TEXT, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%\n");
assertEquals("No magic", InterpreterResult.Type.TEXT, result.type());
}
}
@Test
public void testSimpleMagicType() {
InterpreterResult result = null;
@Test
public void testSimpleMagicData() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table col1\tcol2\naaa\t123\n");
assertEquals("%table col1\tcol2\naaa\t123\n","col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table\ncol1\tcol2\naaa\t123\n");
assertEquals("%table\ncol1\tcol2\naaa\t123\n","col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before magic word %table col1\tcol2\naaa\t123\n");
assertEquals("some text before magic word %table col1\tcol2\naaa\t123\n","col1\tcol2\naaa\t123\n", result.message());
}
@Test
public void testComplexMagicData() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before %table col1\tcol2\naaa\t123\n");
assertEquals("text before %table return %table","col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%html <h3> This is a hack </h3> %table\ncol1\tcol2\naaa\t123\n");
assertEquals("%html before %table return %html"," <h3> This is a hack </h3> %table\n" +
"col1\tcol2\n" +
"aaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"some text before magic word %table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3>");
assertEquals("text & %table befoe %html return %table","col1\tcol2\n" +
"aaa\t123\n" +
" %html <h3> This is a hack </h3>", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3> %table col1\naaa\n123\n");
assertEquals("%table, %html before %table return first %table","col1\tcol2\n" +
"aaa\t123\n" +
" %html <h3> This is a hack </h3> %table col1\n" +
"aaa\n" +
"123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS,"%table col1\tcol2\naaa\t123\n %table col1\naaa\n123\n");
assertEquals("%table before %table return first %table","col1\tcol2\n" +
"aaa\t123\n" +
" %table col1\n" +
"aaa\n" +
"123\n", result.message());
}
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table\ncol1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word %table col1\tcol2\naaa\t123\n");
assertEquals(InterpreterResult.Type.TABLE, result.type());
}
@Test
public void testToString() {
assertEquals("%html hello", new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html hello").toString());
}
public void testComplexMagicType() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before %table col1\tcol2\naaa\t123\n");
assertEquals("some text before magic return magic", InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html <h3> This is a hack </h3> %table\n col1\tcol2\naaa\t123\n");
assertEquals("magic A before magic B return magic A", InterpreterResult.Type.HTML, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word %table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3>");
assertEquals("text & magic A before magic B return magic A", InterpreterResult.Type.TABLE, result.type());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3> %table col1\naaa\n123\n");
assertEquals("magic A, magic B, magic a' return magic A", InterpreterResult.Type.TABLE, result.type());
}
@Test
public void testSimpleMagicData() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n");
assertEquals("%table col1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table\ncol1\tcol2\naaa\t123\n");
assertEquals("%table\ncol1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word %table col1\tcol2\naaa\t123\n");
assertEquals("some text before magic word %table col1\tcol2\naaa\t123\n", "col1\tcol2\naaa\t123\n", result.message());
}
@Test
public void testComplexMagicData() {
InterpreterResult result = null;
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before %table col1\tcol2\naaa\t123\n");
assertEquals("text before %table return %table", "col1\tcol2\naaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html <h3> This is a hack </h3> %table\ncol1\tcol2\naaa\t123\n");
assertEquals("%html before %table return %html", " <h3> This is a hack </h3> %table\n" +
"col1\tcol2\n" +
"aaa\t123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "some text before magic word %table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3>");
assertEquals("text & %table befoe %html return %table", "col1\tcol2\n" +
"aaa\t123\n" +
" %html <h3> This is a hack </h3>", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n %html <h3> This is a hack </h3> %table col1\naaa\n123\n");
assertEquals("%table, %html before %table return first %table", "col1\tcol2\n" +
"aaa\t123\n" +
" %html <h3> This is a hack </h3> %table col1\n" +
"aaa\n" +
"123\n", result.message());
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "%table col1\tcol2\naaa\t123\n %table col1\naaa\n123\n");
assertEquals("%table before %table return first %table", "col1\tcol2\n" +
"aaa\t123\n" +
" %table col1\n" +
"aaa\n" +
"123\n", result.message());
}
@Test
public void testToString() {
assertEquals("%html hello", new InterpreterResult(InterpreterResult.Code.SUCCESS, "%html hello").toString());
}
}

235
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java Normal file
View file

@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.After;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;
private static final int NUM_CLUBBED_EVENTS = 100;
private static final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
private static ScheduledFuture<?> future = null;
/* It is being accessed by multiple threads.
* While loop for 'loopForBufferCompletion' could
* run for-ever.
*/
private volatile static int numInvocations = 0;
@After
public void afterEach() {
if (future != null) {
future.cancel(true);
}
}
@Test
public void testSingleEvent() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String[][] buffer = {{"note", "para", "data\n"}};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend("note", "para", "data\n");
}
@Test
public void testMultipleEventsOfSameParagraph() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String para1 = "para1";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para1, "data2\n"},
{note1, para1, "data3\n"}
};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\ndata2\ndata3\n");
}
@Test
public void testMultipleEventsOfDifferentParagraphs() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String note2 = "note2";
String para1 = "para1";
String para2 = "para2";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para2, "data2\n"},
{note2, para1, "data3\n"},
{note2, para2, "data4\n"}
};
loopForCompletingEvents(listener, 4, buffer);
verify(listener, times(4)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\n");
verify(listener, times(1)).onOutputAppend(note1, para2, "data2\n");
verify(listener, times(1)).onOutputAppend(note2, para1, "data3\n");
verify(listener, times(1)).onOutputAppend(note2, para2, "data4\n");
}
@Test
public void testClubbedData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
Thread thread = new Thread(new BombardEvents(runner));
thread.start();
thread.join();
Thread.sleep(1000);
/* NUM_CLUBBED_EVENTS is a heuristic number.
* It has been observed that for 10,000 continuos event
* calls, 30-40 Web-socket calls are made. Keeping
* the unit-test to a pessimistic 100 web-socket calls.
*/
verify(listener, atMost(NUM_CLUBBED_EVENTS)).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
@Test
public void testWarnLoggerForLargeData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
String data = "data\n";
int numEvents = 100000;
for (int i=0; i<numEvents; i++) {
runner.appendBuffer("noteId", "paraId", data);
}
TestAppender appender = new TestAppender();
Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
Logger.getLogger(RemoteInterpreterEventPoller.class);
runner.run();
List<LoggingEvent> log;
int warnLogCounter;
LoggingEvent sizeWarnLogEntry = null;
do {
warnLogCounter = 0;
log = appender.getLog();
for (LoggingEvent logEntry: log) {
if (Level.WARN.equals(logEntry.getLevel())) {
sizeWarnLogEntry = logEntry;
warnLogCounter += 1;
}
}
} while(warnLogCounter != 2);
String loggerString = "Processing size for buffered append-output is high: " +
(data.length() * numEvents) + " characters.";
assertTrue(loggerString.equals(sizeWarnLogEntry.getMessage()));
}
private class BombardEvents implements Runnable {
private final AppendOutputRunner runner;
private BombardEvents(AppendOutputRunner runner) {
this.runner = runner;
}
@Override
public void run() {
String noteId = "noteId";
String paraId = "paraId";
for (int i=0; i<NUM_EVENTS; i++) {
runner.appendBuffer(noteId, paraId, "data\n");
}
}
}
private class TestAppender extends AppenderSkeleton {
private final List<LoggingEvent> log = new ArrayList<>();
@Override
public boolean requiresLayout() {
return false;
}
@Override
protected void append(final LoggingEvent loggingEvent) {
log.add(loggingEvent);
}
@Override
public void close() {
}
public List<LoggingEvent> getLog() {
return new ArrayList<>(log);
}
}
private void prepareInvocationCounts(RemoteInterpreterProcessListener listener) {
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
numInvocations += 1;
return null;
}
}).when(listener).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
private void loopForCompletingEvents(RemoteInterpreterProcessListener listener,
int numTimes, String[][] buffer) {
numInvocations = 0;
prepareInvocationCounts(listener);
AppendOutputRunner runner = new AppendOutputRunner(listener);
for (String[] bufferElement: buffer) {
runner.appendBuffer(bufferElement[0], bufferElement[1], bufferElement[2]);
}
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
long startTimeMs = System.currentTimeMillis();
while(numInvocations != numTimes) {
if (System.currentTimeMillis() - startTimeMs > 2000) {
fail("Buffered events were not sent for 2 seconds");
}
}
}
}

21
zeppelin-server/pom.xml
View file

@ -357,27 +357,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>src/test/java/com/webautomation/*</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>

96
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -319,7 +319,10 @@ public class NotebookRestApi {
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
String newNoteName = request.getName();
String newNoteName = null;
if (request != null) {
newNoteName = request.getName();
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
@ -567,7 +570,7 @@ public class NotebookRestApi {
}
/**
* Run paragraph job REST API
* Run asynchronously paragraph job REST API
*
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
@ -580,7 +583,7 @@ public class NotebookRestApi {
public Response runParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId, String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
LOG.info("run paragraph job asynchronously {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -593,22 +596,60 @@ public class NotebookRestApi {
}
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.setLastReplName(paragraph.getId());
note.persist(subject);
}
}
handleParagraphParams(message, note, paragraph);
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
/**
* Run synchronously a paragraph REST API
*
* @param noteId - noteId
* @param paragraphId - paragraphId
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("run/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraphSynchronously(@PathParam("notebookId") String noteId,
@PathParam("paragraphId") String paragraphId,
String message) throws
IOException, IllegalArgumentException {
LOG.info("run paragraph synchronously {} {} {}", noteId, paragraphId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
// handle params if presented
handleParagraphParams(message, note, paragraph);
if (paragraph.getListener() == null) {
note.initializeJobListenerForParagraph(paragraph);
}
paragraph.run();
final InterpreterResult result = paragraph.getResult();
if (result.code() == InterpreterResult.Code.SUCCESS) {
return new JsonResponse<>(Status.OK, result).build();
} else {
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, result).build();
}
}
/**
* Stop(delete) paragraph job REST API
*
@ -669,7 +710,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", request.getCronString());
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -697,7 +738,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -738,7 +779,8 @@ public class NotebookRestApi {
LOG.info("Get notebook jobs for job manager");
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
List<Map<String, Object>> notebookJobs = notebook.getJobListforNotebook(false, 0, subject);
List<Map<String, Object>> notebookJobs = notebook
.getJobListByUnixTime(false, 0, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -750,6 +792,8 @@ public class NotebookRestApi {
/**
* Get updated notebook jobs for job manager
*
* Return the `Note` change information within the post unix timestamp.
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@ -763,7 +807,7 @@ public class NotebookRestApi {
List<Map<String, Object>> notebookJobs;
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
notebookJobs = notebook.getJobListforNotebook(false, lastUpdateUnixTime, subject);
notebookJobs = notebook.getJobListByUnixTime(false, lastUpdateUnixTime, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -800,4 +844,20 @@ public class NotebookRestApi {
return new JsonResponse<>(Status.OK, notebooksFound).build();
}
private void handleParagraphParams(String message, Note note, Paragraph paragraph)
throws IOException {
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
}
}
}
}

1
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -104,6 +104,7 @@ public class ZeppelinServer extends Application {
heliumApplicationFactory.setApplicationEventListener(notebookWsServer);
notebook.addNotebookEventListener(heliumApplicationFactory);
notebook.addNotebookEventListener(notebookWsServer.getNotebookInformationListener());
}
public static void main(String[] args) throws InterruptedException {

228
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -241,8 +241,8 @@ public class NotebookServer extends WebSocketServlet implements
case LIST_NOTEBOOK_JOBS:
unicastNotebookJobInfo(conn, messagereceived);
break;
case LIST_UPDATE_NOTEBOOK_JOBS:
unicastUpdateNotebookJobInfo(conn, messagereceived);
case UNSUBSCRIBE_UPDATE_NOTEBOOK_JOBS:
unsubscribeNotebookJobInfo(conn);
break;
case GET_INTERPRETER_BINDINGS:
getInterpreterBindings(conn, messagereceived);
@ -336,7 +336,7 @@ public class NotebookServer extends WebSocketServlet implements
List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.id(), m);
broadcast(note.getId(), m);
}
}
}
@ -398,9 +398,10 @@ public class NotebookServer extends WebSocketServlet implements
}
public void unicastNotebookJobInfo(NotebookSocket conn, Message fromMessage) throws IOException {
addConnectionToNote(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
List<Map<String, Object>> notebookJobs = notebook().getJobListforNotebook(false, 0, subject);
List<Map<String, Object>> notebookJobs = notebook()
.getJobListByUnixTime(false, 0, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -410,21 +411,25 @@ public class NotebookServer extends WebSocketServlet implements
.put("notebookJobs", response)));
}
public void unicastUpdateNotebookJobInfo(NotebookSocket conn, Message fromMessage)
throws IOException {
double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime");
long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue();
List<Map<String, Object>> notebookJobs;
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebookJobs = notebook().getJobListforNotebook(false, lastUpdateUnixTime, subject);
public void broadcastUpdateNotebookJobInfo(long lastUpdateUnixTime) throws IOException {
List<Map<String, Object>> notebookJobs = new LinkedList<>();
Notebook notebookObject = notebook();
List<Map<String, Object>> jobNotes = null;
if (notebookObject != null) {
jobNotes = notebook().getJobListByUnixTime(false, lastUpdateUnixTime, null);
notebookJobs = jobNotes == null ? notebookJobs : jobNotes;
}
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
response.put("jobs", notebookJobs != null ? notebookJobs : new LinkedList<>());
conn.send(serializeMessage(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS)
.put("notebookRunningJobs", response)));
broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
public void unsubscribeNotebookJobInfo(NotebookSocket conn) {
removeConnectionFromNote(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
}
public void saveInterpreterBindings(NotebookSocket conn, Message fromMessage) {
@ -467,17 +472,16 @@ public class NotebookServer extends WebSocketServlet implements
LOG.error("Fail to reload notes from repository", e);
}
}
List<Note> notes = notebook.getAllNotes();
List<Note> notes = notebook.getAllNotes(subject);
List<Map<String, String>> notesInfo = new LinkedList<>();
for (Note note : notes) {
Map<String, String> info = new HashMap<>();
if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) {
if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNotebookId)) {
continue;
}
info.put("id", note.id());
info.put("id", note.getId());
info.put("name", note.getName());
notesInfo.add(info);
}
@ -486,7 +490,7 @@ public class NotebookServer extends WebSocketServlet implements
}
public void broadcastNote(Note note) {
broadcast(note.id(), new Message(OP.NOTE).put("note", note));
broadcast(note.getId(), new Message(OP.NOTE).put("note", note));
}
public void broadcastInterpreterBindings(String noteId,
@ -544,7 +548,7 @@ public class NotebookServer extends WebSocketServlet implements
notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -568,7 +572,7 @@ public class NotebookServer extends WebSocketServlet implements
userAndRoles, notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -604,7 +608,7 @@ public class NotebookServer extends WebSocketServlet implements
note.setName(name);
note.setConfig(config);
if (cronUpdated) {
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
}
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
@ -643,7 +647,7 @@ public class NotebookServer extends WebSocketServlet implements
}
note.persist(subject);
addConnectionToNote(note.id(), (NotebookSocket) conn);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
broadcastNoteList(subject);
}
@ -697,7 +701,7 @@ public class NotebookServer extends WebSocketServlet implements
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
note.persist(subject);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
private void cloneNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -707,7 +711,7 @@ public class NotebookServer extends WebSocketServlet implements
String name = (String) fromMessage.get("name");
Note newNote = notebook.cloneNote(noteId, name, new AuthenticationInfo(fromMessage.principal));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.id(), (NotebookSocket) conn);
addConnectionToNote(newNote.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
broadcastNoteList(subject);
}
@ -719,7 +723,10 @@ public class NotebookServer extends WebSocketServlet implements
if (fromMessage != null) {
String noteName = (String) ((Map) fromMessage.get("notebook")).get("name");
String noteJson = gson.toJson(fromMessage.get("notebook"));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
AuthenticationInfo subject = null;
if (fromMessage.principal != null) {
subject = new AuthenticationInfo(fromMessage.principal);
}
note = notebook.importNote(noteJson, noteName, subject);
note.persist(subject);
broadcastNote(note);
@ -810,12 +817,12 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(note.id()) == null) {
if (setting.getInterpreterGroup(note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(note.id()).getAngularObjectRegistry();
.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -852,17 +859,17 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(n.id()) == null) {
if (setting.getInterpreterGroup(n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(n.id()).getAngularObjectRegistry();
.getInterpreterGroup(n.getId()).getAngularObjectRegistry();
this.broadcastExcept(
n.id(),
n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", n.id())
.put("noteId", n.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -870,10 +877,10 @@ public class NotebookServer extends WebSocketServlet implements
}
} else { // broadcast to all web session for the note
this.broadcastExcept(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -1130,10 +1137,8 @@ public class NotebookServer extends WebSocketServlet implements
.get("config");
p.setConfig(config);
// if it's the last paragraph, let's add a new one
boolean isTheLastParagraph = note.getLastParagraph().getId()
.equals(p.getId());
note.setLastReplName(paragraphId);
if (!(text.equals(note.getLastInterpreterName() + " ") || Strings.isNullOrEmpty(text)) &&
boolean isTheLastParagraph = note.isLastParagraph(p.getId());
if (!(text.trim().equals(p.getMagic()) || Strings.isNullOrEmpty(text)) &&
isTheLastParagraph) {
note.addParagraph();
}
@ -1149,7 +1154,7 @@ public class NotebookServer extends WebSocketServlet implements
new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()),
ex);
p.setStatus(Status.ERROR);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
}
}
@ -1293,6 +1298,113 @@ public class NotebookServer extends WebSocketServlet implements
broadcast(noteId, msg);
}
/**
* Notebook Information Change event
*/
public static class NotebookInformationListener implements NotebookEventListener {
private NotebookServer notebookServer;
public NotebookInformationListener(NotebookServer notebookServer) {
this.notebookServer = notebookServer;
}
@Override
public void onParagraphRemove(Paragraph p) {
try {
notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException ioe) {
LOG.error("can not broadcast for job manager {}", ioe.getMessage());
}
}
@Override
public void onNoteRemove(Note note) {
try {
notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException ioe) {
LOG.error("can not broadcast for job manager {}", ioe.getMessage());
}
List<Map<String, Object>> notesInfo = new LinkedList<>();
Map<String, Object> info = new HashMap<>();
info.put("notebookId", note.getId());
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
// notebook json object root information.
info.put("isRunningJob", false);
info.put("unixTimeLastRun", 0);
info.put("isRemoved", true);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notesInfo);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
@Override
public void onParagraphCreate(Paragraph p) {
Notebook notebook = notebookServer.notebook();
List<Map<String, Object>> notebookJobs = notebook.getJobListByParagraphId(
p.getId()
);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
@Override
public void onNoteCreate(Note note) {
Notebook notebook = notebookServer.notebook();
List<Map<String, Object>> notebookJobs = notebook.getJobListBymNotebookId(
note.getId()
);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
@Override
public void onParagraphStatusChange(Paragraph p, Status status) {
Notebook notebook = notebookServer.notebook();
List<Map<String, Object>> notebookJobs = notebook.getJobListByParagraphId(
p.getId()
);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
@Override
public void onUnbindInterpreter(Note note, InterpreterSetting setting) {
Notebook notebook = notebookServer.notebook();
List<Map<String, Object>> notebookJobs = notebook.getJobListBymNotebookId(
note.getId()
);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
response.put("jobs", notebookJobs);
notebookServer.broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response));
}
}
/**
* Need description here.
*
@ -1309,7 +1421,7 @@ public class NotebookServer extends WebSocketServlet implements
@Override
public void onProgressUpdate(Job job, int progress) {
notebookServer.broadcast(
note.id(),
note.getId(),
new Message(OP.PROGRESS).put("id", job.getId()).put("progress",
job.progress()));
}
@ -1336,6 +1448,12 @@ public class NotebookServer extends WebSocketServlet implements
}
}
notebookServer.broadcastNote(note);
try {
notebookServer.broadcastUpdateNotebookJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException e) {
LOG.error("can not broadcast for job manager {}", e);
}
}
/**
@ -1376,6 +1494,10 @@ public class NotebookServer extends WebSocketServlet implements
return new ParagraphListenerImpl(this, note);
}
public NotebookEventListener getNotebookInformationListener() {
return new NotebookInformationListener(this);
}
private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException {
List<InterpreterSetting> settings =
notebook().getInterpreterFactory().getInterpreterSettings(note.getId());
@ -1384,15 +1506,15 @@ public class NotebookServer extends WebSocketServlet implements
}
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id())
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId())
.getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.id());
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(note.id()).getId())
.put("noteId", note.id())
intpSetting.getInterpreterGroup(note.getId()).getId())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId())
));
}
@ -1413,7 +1535,7 @@ public class NotebookServer extends WebSocketServlet implements
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) {
if (object.getNoteId() != null && !note.getId().equals(object.getNoteId())) {
continue;
}
@ -1424,11 +1546,11 @@ public class NotebookServer extends WebSocketServlet implements
}
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId()));
}
}
@ -1438,7 +1560,7 @@ public class NotebookServer extends WebSocketServlet implements
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (noteId != null && !note.id().equals(noteId)) {
if (noteId != null && !note.getId().equals(noteId)) {
continue;
}
@ -1446,7 +1568,7 @@ public class NotebookServer extends WebSocketServlet implements
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put(
"noteId", noteId).put("paragraphId", paragraphId));
}

11
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -95,28 +95,21 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
ZeppelinITUtils.sleep(1000, false);
waitForParagraph(1, "READY");
String oldIntpTag = driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created above",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.not(StringUtils.EMPTY));
CoreMatchers.equalTo(StringUtils.EMPTY));
setTextOfParagraph(1, " this is above ");
newPara = driver.findElement(By.xpath(getParagraphXPath(2) + "//div[contains(@class,'new-paragraph')][2]"));
action.moveToElement(newPara).click().build().perform();
waitForParagraph(3, "READY");
String lastIntpTag = driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created below",
driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.not(StringUtils.EMPTY));
CoreMatchers.equalTo(StringUtils.EMPTY));
setTextOfParagraph(3, " this is below ");
collector.checkThat("Compare interpreter name tag", oldIntpTag, CoreMatchers.equalTo(lastIntpTag));
collector.checkThat("The output field of paragraph1 contains",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.equalTo(" this is above "));

2
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -134,7 +134,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
Note note = ZeppelinServer.notebook.createNote(null);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
assertThat(get, isAllowed());
get.addRequestHeader("Origin", "http://localhost");
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {

27
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NotebookAuthorization;
@ -144,6 +145,32 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(note1.getId(), null);
}
@Test
public void testCloneNotebook() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
PostMethod post = httpPost("/notebook/" + note1.getId(), "");
LOG.info("testCloneNotebook response\n" + post.getResponseBodyAsString());
assertThat(post, isCreated());
Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
String clonedNotebookId = (String) resp.get("body");
post.releaseConnection();
GetMethod get = httpGet("/notebook/" + clonedNotebookId);
assertThat(get, isAllowed());
Map<String, Object> resp2 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
assertEquals((String)resp2Body.get("name"), "Note " + clonedNotebookId);
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(clonedNotebookId, null);
}
}

3
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
View file

@ -146,8 +146,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("compare note name", expectedNoteName, newNoteName);
assertEquals("initial paragraph check failed", 3, newNote.getParagraphs().size());
for (Paragraph p : newNote.getParagraphs()) {
if (StringUtils.isEmpty(p.getText()) ||
p.getText().trim().equals(newNote.getLastInterpreterName())) {
if (StringUtils.isEmpty(p.getText())) {
continue;
}
assertTrue("paragraph title check failed", p.getTitle().startsWith("title"));

14
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -79,7 +79,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55", p.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -91,7 +91,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
@ -118,7 +118,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[1] 3", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -141,7 +141,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -172,7 +172,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("10\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -204,7 +204,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p2.getStatus());
assertEquals("10", p2.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -216,7 +216,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isPyspark() && sparkVersionNumber >= 14) {
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {

127
zeppelin-web/CONTRIBUTING.md
View file

@ -1,127 +0,0 @@
# Contributing to Zeppelin-Web
## Dev Mode
When working on Zeppelin's WebApplication, it is recommended to run in dev mode.
For that, start Zeppelin server normally, then use ``./grunt serve`` in _zeppelin-web_ directory.
This will launch a Zeppelin WebApplication on port **9000** that will update on code changes.
## Technologies
Zeppelin WebApplication is using **AngularJS** as main Framework, and **Grunt** and **Bower** as helpers.
So you might want to get familiar with it.
[Here is a good start](http://www.sitepoint.com/kickstart-your-angularjs-development-with-yeoman-grunt-and-bower/)
(There is obviously plenty more ressources to learn)
## Coding style
* We follow mainly the [Google Javascript Guide](https://google-styleguide.googlecode.com/svn/trunk/javascriptguide.xml)
* We use a 2 spaces indentation
* We use single quotes
But don't worry, Eslint and Jscs will make you remember it for the most part.
We try not to have **JQuery except in directives**, If you want to include a library,
please search for its **angularJS** directive first.
If you still need to use it, then please use ``angular.element()`` instead of ``$()``
## Folder Structure & Code Organization
* `src` folder: Contains the Source code for Zeppelin WebApplication
* `dist` folder: Contains the compiled code after using **grunt build**
### Src and Code Organization
The `src` folder is organized as such:
<pre>
src/
├── app/
│ ├── name/
│ │ ├── name.controller.js
| | ├── name.html
| | ├── subComponent1/
| | | ├── subComponent1.html
| | | ├── subComponent1.css
│ | | └── subComponent1.controller.js
│ │ └── name.css
│ └── app.js
├── assets/
│ ├── images/
│ └── styles/
| ├── looknfeel/
│ └── printMode.css
├── components/
│ ├── component1/
| | ├── component1.html
│ | └── component1.controller.js
│ └── component2/
├── fonts/
| ├── *.{eot,svg,ttf,woff,otf}
│ └── *.css
├── favico.ico
├── index.html
└── 404.html
</pre>
The code is now organized in a component type of architecture, where everything is logically grouped.
#### File type name convention
In order to understand what is contained inside the .js files without opening it, we use some name conventions:
* .controller.js
* .directive.js
* .service.js
### Component Architecture
When we talk about Component architecture, we think about grouping files together in a logical way.
A component can then be made of multiple files like `.html`, `.css` or any other file type mentioned above.
Related components can be grouped as sub-component as long as they are used in that component only.
#### App folder
Contains the application `app.js` and page related components.
* Home Page
* Interpreter Page
* Notebook Page
etc...
The only resctiction being that a component in the `app` folder is **not used anywhere else**
#### Components folder
The `components` folder is here to contains any reusable component (used more than once)
### Fonts
Fonts files and their css are mixed together in the `fonts` folder
## New files includes
As we do not use yeoman to generate controllers or other type of files with this new structure,
we need to do some includes manually in `index.html` in order to use dev mode and compile correctly.
* Non-bower `.js` files needs to be injected between the tag `<!-- build:js({.tmp,src}) scripts/scripts.js -->`
* Css files needs to be injected between the tag `<!-- build:css(.tmp) styles/main.css -->`
## Add plugins with Bower
```
bower install <plugin> --save
```
The file index.html will automatically update with the new bower_component
<br/>
**Example**: `./bower install angular-nvd3`
You should find that line in the index.html file
```
<script src="bower_components/angular-nvd3/dist/angular-nvd3.js"></script>
````

2
zeppelin-web/README.md
View file

@ -61,4 +61,4 @@ also try to add proxy info to npm install command:
and retry to build again.
## Contribute on Zeppelin Web
If you wish to help us and contribute to Zeppelin WebApplication, please look at [Zeppelin WebApplication's contribution guideline](CONTRIBUTING.md).
If you wish to help us and contribute to Zeppelin WebApplication, please look at the overall project [contribution guidelines](https://zeppelin.apache.org/contribution/contributions.html) and the more focused [Zeppelin WebApplication's documentation](https://zeppelin.apache.org/contribution/webapplication.html).

9
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -325,6 +325,15 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
return;
}
if ($scope.newInterpreterSetting.name.indexOf('.') >= 0) {
BootstrapDialog.alert({
closable: true,
title: 'Add interpreter',
message: '\'.\' is invalid for interpreter name'
});
return;
}
if (_.findIndex($scope.interpreterSettings, {'name': $scope.newInterpreterSetting.name}) >= 0) {
BootstrapDialog.alert({
closable: true,

5
zeppelin-web/src/app/interpreter/interpreter.css
View file

@ -109,3 +109,8 @@
color: #000;
opacity: .5;
}
.remove-margin-top-bottom {
margin-top: 0px !important;
margin-bottom: 0px !important;
}

69
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -38,13 +38,17 @@ limitations under the License.
</div>
</div>
<div class="input-group col-md-4" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters"/>
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
<div class="row">
<div class="col-md-4">
<div class="input-group" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters"/>
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
</div>
</div>
</div>
</div>
@ -93,10 +97,9 @@ limitations under the License.
<small>
<span style="display:inline-block" ng-repeat="interpreter in setting.interpreterGroup"
title="{{interpreter.class}}">
<span ng-show="!$first">, </span>
%<span ng-show="!$parent.$first || $first">{{setting.name}}</span
><span ng-show="(!$parent.$first || $first) && !$first">.</span
><span ng-show="!$first">{{interpreter.name}}</span>
<span ng-show="!$first">, </span>%<span ng-show="!$parent.$first || $first">{{setting.name}}</span>
<span ng-show="(!$parent.$first || $first) && !$first">.</span>
<span ng-show="!$first">{{interpreter.name}}</span>
<span ng-show="$parent.$first && $first">(default)</span>
</span>
</small>
@ -170,37 +173,44 @@ limitations under the License.
</span>
<span>Interpreter for note</span>
</div>
<br />
</div>
<div class="row interpreter" style="margin-top: 5px;">
<div class="col-md-12">
<div class="checkbox">
<div class="checkbox remove-margin-top-bottom">
<span class="input-group" style="line-height:30px;">
<label><input type="checkbox" style="width:20px" id="isExistingProcess" ng-model="setting.option.isExistingProcess" ng-disabled="!valueform.$visible"/>
Connect to existing process </label>
<label>
<input type="checkbox" style="width:20px" id="isExistingProcess" ng-model="setting.option.isExistingProcess" ng-disabled="!valueform.$visible"/>
Connect to existing process
</label>
</span>
</div>
</div>
<div class="col-md-12" ng-show="setting.option.isExistingProcess">
</div>
<div class="row interpreter" ng-if="setting.option.isExistingProcess">
<div class="col-md-12">
<b>Host</b>
<input id="newInterpreterSettingHost" input pu-elastic-input
pu-elastic-input-minwidth="180px" ng-model="setting.option.host" ng-disabled="!valueform.$visible" />
</div>
<div class="col-md-12" ng-show="setting.option.isExistingProcess">
<div class="col-md-12">
<b>Port</b>
<input id="newInterpreterSettingPort" input pu-elastic-input
pu-elastic-input-minwidth="180px" ng-model="setting.option.port" ng-disabled="!valueform.$visible" />
</div>
</div>
<div class="row interpreter">
<div class="col-md-12">
<div class="checkbox">
<div class="checkbox remove-margin-top-bottom">
<span class="input-group" style="line-height:30px;">
<label><input type="checkbox" style="width:18px !important" id="idShowPermission" ng-click="togglePermissions(setting.name)" ng-model="setting.option.setPermission" ng-disabled="!valueform.$visible"/>
Set permission </label>
<label>
<input type="checkbox" style="width:20px !important" id="idShowPermission" ng-click="togglePermissions(setting.name)" ng-model="setting.option.setPermission" ng-disabled="!valueform.$visible"/>
Set permission
</label>
</span>
</div>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12">
<!-- permissions -->
<div ng-show="setting.option.setPermission" class="permissionsForm">
@ -223,14 +233,16 @@ limitations under the License.
<div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
<em>Currently there are no properties and dependencies set for this interpreter</em>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12" ng-show="!_.isEmpty(setting.properties) || valueform.$visible">
<h5>Properties</h5>
<table class="table table-striped">
<thead>
<tr>
<th style="width:30%">name</th>
<th>value</th>
<th ng-if="valueform.$visible">action</th>
<th style="width:40%">name</th>
<th style="width:40%">value</th>
<th style="width:20%" ng-if="valueform.$visible">action</th>
</tr>
</thead>
<tr ng-repeat="key in setting.properties | sortByKey" >
@ -263,7 +275,8 @@ limitations under the License.
</tr>
</table>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12" ng-show="!_.isEmpty(setting.dependencies) || valueform.$visible">
<h5>Dependencies</h5>
<p class="gray40-message" style="font-size:12px" ng-show="valueform.$visible">

6
zeppelin-web/src/app/jobmanager/jobmanager.controller.js
View file

@ -32,14 +32,8 @@ angular.module('zeppelinWebApp')
$scope.JobInfomationsByFilter = $scope.jobInfomations;
websocketMsgSrv.getNotebookJobsList();
var refreshObj = $interval(function() {
if ($scope.lastJobServerUnixTime !== undefined) {
websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime);
}
}, 1000);
$scope.$on('$destroy', function() {
$interval.cancel(refreshObj);
websocketMsgSrv.unsubscribeJobManager();
});
};

2
zeppelin-web/src/app/notebook/notebook.html
View file

@ -26,7 +26,7 @@ limitations under the License.
Bind interpreter for this note.
Click to Bind/Unbind interpreter.
Drag and drop to reorder interpreters. <br />
The first interpreter on the list becomes default. To create/remove interpreters, go to <a href="/#/interpreter">Interpreter</a> menu.
The first interpreter on the list becomes default. To create/remove interpreters, go to <a href="#/interpreter">Interpreter</a> menu.
</p>
<div class="interpreterSettings"

37
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -820,7 +820,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
var user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user;
var desc = 'Took ' + moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') +
'. Last updated by ' + user + ' at ' + moment(pdata.dateUpdated).format('MMMM DD YYYY, h:mm:ss A') + '.';
'. Last updated by ' + user + ' at ' + moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + '.';
if ($scope.isResultOutdated()) {
desc += ' (outdated)';
}
@ -865,6 +865,21 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
};
$scope.parseTableCell = function(cell) {
if (!isNaN(cell)) {
if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) {
return cell;
} else {
return Number(cell);
}
}
var d = moment(cell);
if (d.isValid()) {
return d;
}
return cell;
};
$scope.loadTableData = function(result) {
if (!result) {
return;
@ -898,8 +913,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
if (i === 0) {
columnNames.push({name: col, index: j, aggr: 'sum'});
} else {
cols.push(col);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: col});
var parsedCol = $scope.parseTableCell(col);
cols.push(parsedCol);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: parsedCol});
}
}
if (i !== 0) {
@ -978,7 +994,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
cells: function(row, col, prop) {
var cellProperties = {};
cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) {
if (!isNaN(value)) {
if (value instanceof moment) {
td.innerHTML = value._i;
} else if (!isNaN(value)) {
cellProperties.format = '0,0.[00000]';
td.style.textAlign = 'left';
Handsontable.renderers.NumericRenderer.apply(this, arguments);
@ -2575,7 +2593,16 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
});
$scope.$on('appendParagraphOutput', function(event, data) {
if ($scope.paragraph.id === data.paragraphId) {
/* It has been observed that append events
* can be errorneously called even if paragraph
* execution has ended, and in that case, no append
* should be made. Also, it was observed that between PENDING
* and RUNNING states, append-events can be called and we can't
* miss those, else during the length of paragraph run, few
* initial output line/s will be missing.
*/
if ($scope.paragraph.id === data.paragraphId &&
($scope.paragraph.status === 'RUNNING' || $scope.paragraph.status === 'PENDING')) {
if ($scope.flushStreamingOutput) {
$scope.clearTextOutput();
$scope.flushStreamingOutput = false;

1
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -502,7 +502,6 @@ div.esri-view {
table.table-striped {
border-top: 1px solid #ddd;
margin-top: 20px;
}
.scroll-paragraph-down {

2
zeppelin-web/src/components/websocketEvents/websocketMsg.service.js
View file

@ -195,7 +195,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope,
},
unsubscribeJobManager: function() {
websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_JOBMANAGER'});
websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_UPDATE_NOTEBOOK_JOBS'});
},
getInterpreterBindings: function(noteID) {

15
zeppelin-zengine/pom.xml
View file

@ -128,12 +128,6 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
@ -258,5 +252,14 @@
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration combine.children="append">
<forkMode>always</forkMode>
</configuration>
</plugin>
</plugins>
</build>
</project>

30
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -272,10 +272,15 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getKeyStorePath() {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH)));
String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
public String getKeyStoreType() {
@ -297,10 +302,13 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getTrustStorePath() {
String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
if (path == null) {
return getKeyStorePath();
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(path);
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
@ -427,10 +435,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
public boolean getUseJdbcAlias() {
return getBoolean(ConfVars.ZEPPELIN_USE_JDBC_ALIAS);
}
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@ -557,9 +561,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true);
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000");
private String varName;
@SuppressWarnings("rawtypes")

90
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -171,32 +171,42 @@ public class InterpreterFactory implements InterpreterGroupFactory {
})) {
String interpreterDirString = interpreterDir.toString();
registerInterpreterFromPath(interpreterDirString, interpreterJson);
registerInterpreterFromResource(cl, interpreterDirString, interpreterJson);
/*
* TODO(jongyoul)
* - Remove these codes below because of legacy code
* - Support ThreadInterpreter
/**
* Register interpreter by the following ordering
* 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
* interpreter-setting.json
* 2. Register it from interpreter-setting.json in classpath
* {ZEPPELIN_HOME}/interpreter/{interpreter_name}
* 3. Register it by Interpreter.register
*/
URLClassLoader ccl = new URLClassLoader(recursiveBuildLibList(interpreterDir.toFile()), cl);
for (String className : interpreterClassList) {
try {
// Load classes
Class.forName(className, true, ccl);
Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
for (String interpreterKey : interpreterKeys) {
if (className
.equals(Interpreter.registeredInterpreters.get(interpreterKey).getClassName())) {
Interpreter.registeredInterpreters.get(interpreterKey)
.setPath(interpreterDirString);
logger.info("Interpreter " + interpreterKey + " found. class=" + className);
cleanCl.put(interpreterDirString, ccl);
if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
/*
* TODO(jongyoul)
* - Remove these codes below because of legacy code
* - Support ThreadInterpreter
*/
URLClassLoader ccl = new URLClassLoader(
recursiveBuildLibList(interpreterDir.toFile()), cl);
for (String className : interpreterClassList) {
try {
// Load classes
Class.forName(className, true, ccl);
Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
for (String interpreterKey : interpreterKeys) {
if (className
.equals(Interpreter.registeredInterpreters.get(interpreterKey)
.getClassName())) {
Interpreter.registeredInterpreters.get(interpreterKey)
.setPath(interpreterDirString);
logger.info("Interpreter " + interpreterKey + " found. class=" + className);
cleanCl.put(interpreterDirString, ccl);
}
}
} catch (Throwable t) {
// nothing to do
}
}
} catch (Throwable t) {
// nothing to do
}
}
}
@ -277,7 +287,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return properties;
}
private void registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
String interpreterJson) throws IOException, RepositoryException {
URL[] urls = recursiveBuildLibList(new File(interpreterDir));
ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
@ -289,10 +299,12 @@ public class InterpreterFactory implements InterpreterGroupFactory {
List<RegisteredInterpreter> registeredInterpreterList =
getInterpreterListFromJson(inputStream);
registerInterpreters(registeredInterpreterList, interpreterDir);
return true;
}
return false;
}
private void registerInterpreterFromPath(String interpreterDir, String interpreterJson)
private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
throws IOException, RepositoryException {
Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
@ -301,7 +313,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
List<RegisteredInterpreter> registeredInterpreterList =
getInterpreterListFromJson(interpreterJsonPath);
registerInterpreters(registeredInterpreterList, interpreterDir);
return true;
}
return false;
}
private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename)
@ -506,6 +520,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
public InterpreterSetting createNewSetting(String name, String group,
List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
if (name.indexOf(".") >= 0) {
throw new IOException("'.' is invalid for InterpreterSetting name.");
}
InterpreterSetting setting = createFromInterpreterSettingRef(group);
setting.setName(name);
setting.setGroup(group);
@ -669,7 +686,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
Interpreter interpreter;
for (InterpreterInfo info : interpreterInfos) {
if (option.isRemote()) {
if (option.isConnectExistingProcess()) {
if (option.isExistingProcess()) {
interpreter =
connectToRemoteRepl(noteId, info.getClassName(), option.getHost(), option.getPort(),
properties);
@ -991,7 +1008,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
new RemoteInterpreter(property, noteId, className, conf.getInterpreterRemoteRunnerPath(),
interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
remoteInterpreterProcessListener, appEventListener);
remoteInterpreter.setEnv(env);
remoteInterpreter.addEnv(env);
return new LazyOpenInterpreter(remoteInterpreter);
}
@ -1014,15 +1031,16 @@ public class InterpreterFactory implements InterpreterGroupFactory {
public List<InterpreterSetting> getInterpreterSettings(String noteId) {
List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
LinkedList<InterpreterSetting> settings = new LinkedList<>();
synchronized (interpreterSettingIds) {
for (String id : interpreterSettingIds) {
InterpreterSetting setting = get(id);
if (setting == null) {
// interpreter setting is removed from factory. remove id from here, too
interpreterSettingIds.remove(id);
} else {
settings.add(setting);
}
Iterator<String> iter = interpreterSettingIds.iterator();
while (iter.hasNext()) {
String id = iter.next();
InterpreterSetting setting = get(id);
if (setting == null) {
// interpreter setting is removed from factory. remove id from here, too
iter.remove();
} else {
settings.add(setting);
}
}
return settings;

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java
View file

@ -85,10 +85,6 @@ public class InterpreterOption {
this.perNoteSession = perNoteSession;
}
public boolean isConnectExistingProcess() {
return (host != null && port != -1);
}
public String getHost() {
return host;
}

136
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
@ -27,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import com.google.gson.Gson;
import org.apache.commons.lang.StringUtils;
@ -76,7 +77,6 @@ public class Note implements Serializable, ParagraphJobListener {
private String name = "";
private String id;
private AtomicReference<String> lastReplName = new AtomicReference<>(StringUtils.EMPTY);
private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create();
private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
@ -125,16 +125,6 @@ public class Note implements Serializable, ParagraphJobListener {
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
void putDefaultReplName() {
String defaultInterpreterName = getDefaultInterpreterName();
logger.info("defaultInterpreterName is '{}'", defaultInterpreterName);
lastReplName.set(defaultInterpreterName);
}
public String id() {
return id;
}
public String getId() {
return id;
}
@ -172,6 +162,28 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (paragraphNote.getId().equals(this.getId())) {
throw new IllegalArgumentException(format("The paragraph %s from note %s " +
"does not belong to note %s", paragraph.getId(), paragraphNote.getId(),
this.getId()));
}
boolean foundParagraph = false;
for (Paragraph ownParagraph : paragraphs) {
if (paragraph.getId().equals(ownParagraph.getId())) {
paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
foundParagraph = true;
}
}
if (!foundParagraph) {
throw new IllegalArgumentException(format("Cannot find paragraph %s " +
"from note %s", paragraph.getId(), paragraphNote.getId()));
}
}
void setJobListenerFactory(JobListenerFactory jobListenerFactory) {
this.jobListenerFactory = jobListenerFactory;
}
@ -202,7 +214,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph addParagraph() {
Paragraph p = new Paragraph(this, this, factory);
addLastReplNameIfEmptyText(p);
setParagraphMagic(p, paragraphs.size());
synchronized (paragraphs) {
paragraphs.add(p);
}
@ -258,7 +270,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph insertParagraph(int index) {
Paragraph p = new Paragraph(this, this, factory);
addLastReplNameIfEmptyText(p);
setParagraphMagic(p, index);
synchronized (paragraphs) {
paragraphs.add(index, p);
}
@ -268,22 +280,6 @@ public class Note implements Serializable, ParagraphJobListener {
return p;
}
/**
* Add Last Repl name If Paragraph has empty text
*
* @param p Paragraph
*/
private void addLastReplNameIfEmptyText(Paragraph p) {
String replName = lastReplName.get();
if (StringUtils.isEmpty(p.getText()) && StringUtils.isNotEmpty(replName)) {
p.setText(getInterpreterName(replName) + " ");
}
}
private String getInterpreterName(String replName) {
return StringUtils.isBlank(replName) ? StringUtils.EMPTY : "%" + replName;
}
/**
* Remove paragraph by id.
*
@ -292,7 +288,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph removeParagraph(String paragraphId) {
removeAllAngularObjectInParagraph(paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(id(), paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@ -354,8 +350,8 @@ public class Note implements Serializable, ParagraphJobListener {
if (index < 0 || index >= paragraphs.size()) {
if (throwWhenIndexIsOutOfBound) {
throw new IndexOutOfBoundsException("paragraph size is " + paragraphs.size() +
" , index is " + index);
throw new IndexOutOfBoundsException(
"paragraph size is " + paragraphs.size() + " , index is " + index);
} else {
return;
}
@ -411,7 +407,7 @@ public class Note implements Serializable, ParagraphJobListener {
List<Map<String, String>> paragraphsInfo = new LinkedList<>();
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
Map<String, String> info = populatePragraphInfo(p);
Map<String, String> info = populateParagraphInfo(p);
paragraphsInfo.add(info);
}
}
@ -422,14 +418,14 @@ public class Note implements Serializable, ParagraphJobListener {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return populatePragraphInfo(p);
return populateParagraphInfo(p);
}
}
return new HashMap<>();
}
}
private Map<String, String> populatePragraphInfo(Paragraph p) {
private Map<String, String> populateParagraphInfo(Paragraph p) {
Map<String, String> info = new HashMap<>();
info.put("id", p.getId());
info.put("status", p.getStatus().toString());
@ -445,15 +441,26 @@ public class Note implements Serializable, ParagraphJobListener {
return info;
}
private void setParagraphMagic(Paragraph p, int index) {
if (paragraphs.size() > 0) {
String magic;
if (index == 0) {
magic = paragraphs.get(0).getMagic();
} else {
magic = paragraphs.get(index - 1).getMagic();
}
if (StringUtils.isNotEmpty(magic)) {
p.setText(magic + "\n");
}
}
}
/**
* Run all paragraphs sequentially.
*/
public void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
synchronized (paragraphs) {
if (!paragraphs.isEmpty()) {
setLastReplName(paragraphs.get(paragraphs.size() - 1));
}
for (Paragraph p : paragraphs) {
if (!p.isEnabled()) {
continue;
@ -476,27 +483,15 @@ public class Note implements Serializable, ParagraphJobListener {
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(getId(), requiredReplName);
if (intp == null) {
// TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter
if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) {
String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")");
logger.debug("New paragraph: {}", pText);
p.setEffectiveText(pText);
} else {
String intpExceptionMsg = String.format("%s",
p.getJobName()
+ "'s Interpreter "
+ requiredReplName + " not found"
);
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult = new InterpreterResult(
InterpreterResult.Code.ERROR, intpException.getMessage()
);
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
intp.getScheduler().submit(p);
@ -591,16 +586,6 @@ public class Note implements Serializable, ParagraphJobListener {
repo.save(this, subject);
}
private void setLastReplName(Paragraph lastParagraphStarted) {
if (StringUtils.isNotEmpty(lastParagraphStarted.getRequiredReplName())) {
lastReplName.set(lastParagraphStarted.getRequiredReplName());
}
}
public void setLastReplName(String paragraphId) {
setLastReplName(getParagraph(paragraphId));
}
/**
* Persist this note with maximum delay.
*/
@ -609,7 +594,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
void unpersist(AuthenticationInfo subject) throws IOException {
repo.remove(id(), subject);
repo.remove(getId(), subject);
}
@ -665,14 +650,6 @@ public class Note implements Serializable, ParagraphJobListener {
this.info = info;
}
String getLastReplName() {
return lastReplName.get();
}
public String getLastInterpreterName() {
return getInterpreterName(getLastReplName());
}
@Override
public void beforeStatusChange(Job job, Status before, Status after) {
if (jobListenerFactory != null) {
@ -707,7 +684,6 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
@Override
public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) {
if (jobListenerFactory != null) {

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
View file

@ -36,7 +36,7 @@ public class NoteInfo {
}
public NoteInfo(Note note) {
id = note.id();
id = note.getId();
name = note.getName();
config = note.getConfig();
}

140
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -24,15 +24,21 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonReader;
import org.apache.commons.codec.binary.StringUtils;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
import org.quartz.JobBuilder;
@ -151,13 +157,17 @@ public class Notebook implements NoteEventListener {
Note note =
new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this);
synchronized (notes) {
notes.put(note.id(), note);
notes.put(note.getId(), note);
}
if (interpreterIds != null) {
bindInterpretersToNote(note.id(), interpreterIds);
note.putDefaultReplName();
bindInterpretersToNote(note.getId(), interpreterIds);
}
if (subject != null && !"anonymous".equals(subject.getUser())) {
Set<String> owners = new HashSet<String>();
owners.add(subject.getUser());
notebookAuthorization.setOwners(note.getId(), owners);
}
notebookIndex.addIndexDoc(note);
note.persist(subject);
fireNoteCreateEvent(note);
@ -239,10 +249,12 @@ public class Notebook implements NoteEventListener {
Note newNote = createNote(subject);
if (newNoteName != null) {
newNote.setName(newNoteName);
} else {
newNote.setName("Note " + newNote.getId());
}
// Copy the interpreter bindings
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.id());
bindInterpretersToNote(newNote.id(), boundInterpreterSettingsIds);
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);
List<Paragraph> paragraphs = sourceNote.getParagraphs();
for (Paragraph p : paragraphs) {
@ -419,15 +431,15 @@ public class Notebook implements NoteEventListener {
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.id(), note);
refreshCron(note.id());
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.id());
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
@ -467,7 +479,7 @@ public class Notebook implements NoteEventListener {
if (notebookRepo instanceof NotebookRepoSync) {
NotebookRepoSync mainRepo = (NotebookRepoSync) notebookRepo;
if (mainRepo.getRepoCount() > 1) {
mainRepo.sync();
mainRepo.sync(subject);
}
}
@ -508,11 +520,11 @@ public class Notebook implements NoteEventListener {
Collections.sort(noteList, new Comparator<Note>() {
@Override
public int compare(Note note1, Note note2) {
String name1 = note1.id();
String name1 = note1.getId();
if (note1.getName() != null) {
name1 = note1.getName();
}
String name2 = note2.id();
String name2 = note2.getId();
if (note2.getName() != null) {
name2 = note2.getName();
}
@ -523,6 +535,35 @@ public class Notebook implements NoteEventListener {
}
}
public List<Note> getAllNotes(AuthenticationInfo subject) {
final Set<String> entities = Sets.newHashSet();
if (subject != null) {
entities.add(subject.getUser());
}
synchronized (notes) {
return FluentIterable.from(notes.values()).filter(new Predicate<Note>() {
@Override
public boolean apply(Note input) {
return input != null && notebookAuthorization.isReader(input.getId(), entities);
}
}).toSortedList(new Comparator<Note>() {
@Override
public int compare(Note note1, Note note2) {
String name1 = note1.getId();
if (note1.getName() != null) {
name1 = note1.getName();
}
String name2 = note2.getId();
if (note2.getName() != null) {
name2 = note2.getName();
}
return name1.compareTo(name2);
}
});
}
}
private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
Map<String, Object> paragraphItem = new HashMap<>();
@ -572,7 +613,78 @@ public class Notebook implements NoteEventListener {
return lastRunningUnixTime;
}
public List<Map<String, Object>> getJobListforNotebook(boolean needsReload,
public List<Map<String, Object>> getJobListByParagraphId(String paragraphID) {
String gotNoteId = null;
List<Note> notes = getAllNotes();
for (Note note : notes) {
Paragraph p = note.getParagraph(paragraphID);
if (p != null) {
gotNoteId = note.getId();
}
}
return getJobListBymNotebookId(gotNoteId);
}
public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
long lastRunningUnixTime = 0;
boolean isNotebookRunning = false;
Note jobNote = getNote(notebookID);
List<Map<String, Object>> notesInfo = new LinkedList<>();
if (jobNote == null) {
return notesInfo;
}
Map<String, Object> info = new HashMap<>();
info.put("notebookId", jobNote.getId());
String notebookName = jobNote.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", jobNote.getName());
} else {
info.put("notebookName", "Note " + jobNote.getId());
}
// set notebook type ( cron or normal )
if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) && !jobNote.getConfig()
.get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : jobNote.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(jobNote.getId()) != null
&& replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
return notesInfo;
};
public List<Map<String, Object>> getJobListByUnixTime(boolean needsReload,
long lastUpdateServerUnixTime, AuthenticationInfo subject) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
@ -593,14 +705,14 @@ public class Notebook implements NoteEventListener {
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.id());
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.id());
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )

63
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.notebook;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.helium.HeliumPackage;
@ -53,7 +54,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private transient InterpreterFactory factory;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient String effectiveText;
String title;
String text;
@ -114,14 +114,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
this.dateUpdated = new Date();
}
public void setEffectiveText(String effectiveText) {
this.effectiveText = effectiveText;
}
public String getEffectiveText() {
return effectiveText;
}
public AuthenticationInfo getAuthenticationInfo() {
return authenticationInfo;
}
@ -153,7 +145,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getRequiredReplName() {
return getRequiredReplName(null != effectiveText ? effectiveText : text);
return getRequiredReplName(text);
}
public static String getRequiredReplName(String text) {
@ -182,7 +174,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getScriptBody() {
return getScriptBody(null != effectiveText ? effectiveText : text);
return getScriptBody(text);
}
public static String getScriptBody(String text) {
@ -359,7 +351,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
} finally {
InterpreterContext.remove();
effectiveText = null;
}
}
@ -451,13 +442,13 @@ public class Paragraph extends Job implements Serializable, Cloneable {
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(note.id()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.id()).getResourcePool();
registry = intpGroup.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
@ -465,22 +456,22 @@ public class Paragraph extends Job implements Serializable, Cloneable {
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials = credentials.getUserCredentials(
authenticationInfo.getUser());
authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext = new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
note.getId(),
getId(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}
@ -575,4 +566,22 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
return scriptBody;
}
public String getMagic() {
String magic = StringUtils.EMPTY;
String text = getText();
if (text != null && text.startsWith("%")) {
magic = text.split("\\s+")[0];
if (isValidInterpreter(magic.substring(1))) {
return magic;
} else {
return StringUtils.EMPTY;
}
}
return magic;
}
private boolean isValidInterpreter(String replName) {
return factory.getInterpreter(note.getId(), replName) != null;
}
}

28
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
View file

@ -93,7 +93,8 @@ public class NotebookRepoSync implements NotebookRepo {
}
if (getRepoCount() > 1) {
try {
sync(0, 1);
AuthenticationInfo subject = new AuthenticationInfo("anonymous");
sync(0, 1, subject);
} catch (IOException e) {
LOG.warn("Failed to sync with secondary storage on start {}", e);
}
@ -175,12 +176,12 @@ public class NotebookRepoSync implements NotebookRepo {
*
* @throws IOException
*/
void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
void sync(int sourceRepoIndex, int destRepoIndex, AuthenticationInfo subject) throws IOException {
LOG.info("Sync started");
NotebookRepo srcRepo = getRepo(sourceRepoIndex);
NotebookRepo dstRepo = getRepo(destRepoIndex);
List <NoteInfo> srcNotes = srcRepo.list(null);
List <NoteInfo> dstNotes = dstRepo.list(null);
List <NoteInfo> srcNotes = srcRepo.list(subject);
List <NoteInfo> dstNotes = dstRepo.list(subject);
Map<String, List<String>> noteIDs = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo);
List<String> pushNoteIDs = noteIDs.get(pushKey);
@ -192,7 +193,7 @@ public class NotebookRepoSync implements NotebookRepo {
for (String id : pushNoteIDs) {
LOG.info("ID : " + id);
}
pushNotes(pushNoteIDs, srcRepo, dstRepo);
pushNotes(subject, pushNoteIDs, srcRepo, dstRepo);
} else {
LOG.info("Nothing to push");
}
@ -202,7 +203,7 @@ public class NotebookRepoSync implements NotebookRepo {
for (String id : pullNoteIDs) {
LOG.info("ID : " + id);
}
pushNotes(pullNoteIDs, dstRepo, srcRepo);
pushNotes(subject, pullNoteIDs, dstRepo, srcRepo);
} else {
LOG.info("Nothing to pull");
}
@ -212,7 +213,7 @@ public class NotebookRepoSync implements NotebookRepo {
for (String id : delDstNoteIDs) {
LOG.info("ID : " + id);
}
deleteNotes(delDstNoteIDs, dstRepo);
deleteNotes(subject, delDstNoteIDs, dstRepo);
} else {
LOG.info("Nothing to delete from dest");
}
@ -220,20 +221,21 @@ public class NotebookRepoSync implements NotebookRepo {
LOG.info("Sync ended");
}
public void sync() throws IOException {
sync(0, 1);
public void sync(AuthenticationInfo subject) throws IOException {
sync(0, 1, subject);
}
private void pushNotes(List<String> ids, NotebookRepo localRepo,
private void pushNotes(AuthenticationInfo subject, List<String> ids, NotebookRepo localRepo,
NotebookRepo remoteRepo) throws IOException {
for (String id : ids) {
remoteRepo.save(localRepo.get(id, null), null);
remoteRepo.save(localRepo.get(id, subject), subject);
}
}
private void deleteNotes(List<String> ids, NotebookRepo repo) throws IOException {
private void deleteNotes(AuthenticationInfo subject, List<String> ids, NotebookRepo repo)
throws IOException {
for (String id : ids) {
repo.remove(id, null);
repo.remove(id, subject);
}
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -210,7 +210,7 @@ public class S3NotebookRepo implements NotebookRepo {
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json";
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -226,7 +226,7 @@ public class VFSNotebookRepo implements NotebookRepo {
FileObject rootDir = getRootDir();
FileObject noteDir = rootDir.resolveFile(note.id(), NameScope.CHILD);
FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD);
if (!noteDir.exists()) {
noteDir.createFolder();

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
View file

@ -176,7 +176,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
String notebook = GSON.toJson(note);
restApiClient.asyncPut(notebook);
LOG.info("ZeppelinHub REST API saving note {} ", note.id());
LOG.info("ZeppelinHub REST API saving note {} ", note.getId());
}
@Override

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
View file

@ -127,8 +127,8 @@ public class Message {
APP_STATUS_CHANGE, // [s-c] on app status change
LIST_NOTEBOOK_JOBS, // [c-s] get notebook job management infomations
LIST_UPDATE_NOTEBOOK_JOBS, // [c-s] get job management informations for until unixtime
// @param unixTime
LIST_UPDATE_NOTEBOOK_JOBS, // [s-c] get job management informations
UNSUBSCRIBE_UPDATE_NOTEBOOK_JOBS, // [c-s] unsubscribe job information for job management
GET_INTERPRETER_BINDINGS, // [c-s] get interpreter bindings
// @param noteID
SAVE_INTERPRETER_BINDINGS, // [c-s] save interpreter bindings

12
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -176,7 +176,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
factory.setInterpreters(note1.id(), factory.getDefaultInterpreterSettingList());
factory.setInterpreters(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -214,7 +214,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -231,7 +231,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
}
// when unbind interpreter
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// then
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
@ -255,7 +255,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// Unbind all interpreter from note
// NullPointerException shouldn't occur here
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// remove note
notebook.removeNote(note1.getId(), null);
@ -273,9 +273,9 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.id())) {
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
mock1IntpSettingId = setting.getId();
break;

41
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -21,6 +21,8 @@ import java.io.*;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
@ -31,6 +33,7 @@ import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -52,7 +55,10 @@ public class InterpreterFactoryTest {
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
Map<String, InterpreterProperty> propertiesMockInterpreter1 = new HashMap<String, InterpreterProperty>();
propertiesMockInterpreter1.put("PROPERTY_1", new InterpreterProperty("PROPERTY_1", "", "VALUE_1", "desc"));
propertiesMockInterpreter1.put("property_2", new InterpreterProperty("", "property_2", "value_2", "desc"));
MockInterpreter1.register("mock1", "mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1", propertiesMockInterpreter1);
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
@ -95,6 +101,29 @@ public class InterpreterFactoryTest {
assertNull(mock1Setting.getInterpreterGroup("sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver);
List<InterpreterSetting> all = factory.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess");
factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
LazyOpenInterpreter lazyInterpreter = (LazyOpenInterpreter)(interpreterGroup.get("session").get(0));
assertTrue(lazyInterpreter.getInnerInterpreter() instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) lazyInterpreter.getInnerInterpreter();
assertEquals("VALUE_1", remoteInterpreter.getEnv().get("PROPERTY_1"));
assertEquals("value_2", remoteInterpreter.getProperty("property_2"));
}
@Test
public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default settings
@ -157,4 +186,14 @@ public class InterpreterFactoryTest {
assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("note", "group1").getClassName());
}
@Test
public void testInvalidInterpreterSettingName() {
try {
factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
}
}
}

Some files were not shown because too many files have changed in this diff Show more