Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-1306

Browse source
This commit is contained in:
astroshim 2016-09-09 15:23:56 +09:00
commit 0453402769
101 changed files with 2206 additions and 1720 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/PULL_REQUEST_TEMPLATE vendored
View file

@ -1,6 +1,6 @@
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md
First time? Check out the contributing guide - https://zeppelin.apache.org/contribution/contributions.html
### What type of PR is it?

4
.gitignore vendored
View file

@ -47,6 +47,7 @@ zeppelin-web/bower_components
# R
/r/lib/
.Rhistory
/R/
# project level
/logs/
@ -108,3 +109,6 @@ tramp
# Generated by zeppelin-examples
/helium
# tmp files
/tmp/

26
.travis.yml
View file

@ -21,6 +21,7 @@ cache:
directories:
- .spark-dist
- ${HOME}/.m2/repository/.cache/maven-download-plugin
- .node_modules
addons:
apt:
@ -33,44 +34,49 @@ addons:
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.0.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.1
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- mkdir -p ~/R
- echo 'R_LIBS=~/R' > ~/.Renviron
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
@ -89,6 +95,7 @@ before_script:
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
- rm -rf .node_modules; cp -r zeppelin-web/node_modules .node_modules
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
@ -104,3 +111,4 @@ after_failure:
after_script:
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER

228
CONTRIBUTING.md
View file

@ -1,228 +0,0 @@
# How to contribute
**Zeppelin** is [Apache2 License](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md) Software.
Contributing to Zeppelin (Source code, Documents, Image, Website) means you agree to the Apache2 License.
1. Make sure your issue is not already in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
3. Contribute your patch via Pull Request.
Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it.
For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful.
## Creating a Pull Request
In order to make the review process easier, please follow this template when making a Pull Request:
```
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md
### What type of PR is it?
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?
```
## Testing a Pull Request
You can also test and review a particular Pull Request. Here are two useful ways.
* Using a utility provided from Zeppelin.
```
dev/test_zeppelin_pr.py [# of PR]
```
For example, if you want to test `#513`, then the command will be:
```
dev/test_zeppelin_pr.py 513
```
* Another way is using [github/hub](https://github.com/github/hub).
```
hub checkout https://github.com/apache/zeppelin/pull/[# of PR]
```
The above two methods will help you test and review Pull Requests.
## Source Control Workflow
Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model.
## The Review Process
When a Pull Request is submitted, it is being merged or rejected by following review process.
* Anybody can be a reviewer and may comment on the change and suggest modifications.
* Reviewer can indicate that a patch looks suitable for merging with a comment such as: "Looks good", "LGTM", "+1".
* At least one indication of suitable for merging (e.g. "LGTM") from committer is required to be merged.
* Pull request is open for 1 or 2 days for potential additional review, unless it's got enough indication of suitable for merging.
* Committer can initiate lazy consensus ("Merge if there is no more discussion") and the code can be merged after certain time (normally 24 hours) when there is no review exists.
* Contributor can ping reviewers (including committer) by commenting 'Ready to review' or suitable indication.
## Becoming a Committer
The PPMC adds new committers from the active contributors, based on their contribution to Zeppelin. The qualifications for new committers include:
1. Sustained contributions: Committers should have a history of constant contributions to Zeppelin.
2. Quality of contributions: Committers more than any other community member should submit simple, well-tested, and well-designed patches.
3. Community involvement: Committers should have a constructive and friendly attitude in all community interactions. They should also be active on the dev, user list and reviewing patches. Also help new contributors and users.
## Setting up
Here are some things you will need to build and test Zeppelin.
### Software Configuration Management (SCM)
Zeppelin uses Git for its SCM system. `http://git.apache.org/zeppelin.git` you'll need git client installed in your development machine.
For write access, `https://git-wip-us.apache.org/repos/asf/zeppelin.git`
### Integrated Development Environment (IDE)
You are free to use whatever IDE you prefer, or your favorite command line editor.
### Project Structure
Zeppelin project is based on Maven. Maven works by convention & defines [directory structure] (https://maven.apache.org/guides/introduction/introduction-to-the-standard-directory-layout.html) for a project.
The top-level pom.xml describes the basic project structure. Currently Zeppelin has the following modules.
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>spark</module>
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>flink</module>
<module>ignite</module>
<module>lens</module>
<module>cassandra</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-distribution</module>
### Web Project Contribution Guidelines
If you plan on making a contribution to Zeppelin's WebApplication,
please check [its own contribution guidelines](https://github.com/apache/zeppelin/blob/master/zeppelin-web/CONTRIBUTING.md)
### Code convention
We are following Google Code style:
* [Java style](https://google.github.io/styleguide/javaguide.html)
* [Shell style](https://google.github.io/styleguide/shell.xml)
Check style report location are in `${submodule}/target/site/checkstyle.html`
Test coverage report location are in `${submodule}/target/site/cobertura/index.html`
#### Build Tools
To build the code, install
* Oracle Java 7
* Apache Maven
## Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git).
### git access
Get the source code on your development machine using git.
```
git clone git://git.apache.org/zeppelin.git zeppelin
```
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin
```
or with write access
```
git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git
```
### Fork repository
If you want not only build Zeppelin but also make change, then you need fork Zeppelin github mirror repository (https://github.com/apache/zeppelin) and make pull request.
## Build
```
mvn install
```
To skip test
```
mvn install -DskipTests
```
To build with specific spark / hadoop version
```
mvn install -Phadoop-2.2 -Dhadoop.version=2.2.0 -Pspark-1.3 -Dspark.version=1.3.0
```
## Tests
Each new File should have its own accompanying unit tests. Each new interpreter should have come with its tests.
Zeppelin has 3 types of tests:
1. Unit Tests: The unit tests run as part of each package's build. E.g. SparkInterpeter Module's unit test is SparkInterpreterTest
2. Integration Tests: The integration tests run after all modules are build. The integration tests launch an instance of Zeppelin server. ZeppelinRestApiTest is an example integration test.
3. GUI integration tests: These tests validate the Zeppelin UI elements. These tests require a running Zeppelin server and launches a web browser to validate Notebook UI elements like Notes and their execution. See ZeppelinIT as an example.
Currently the GUI integration tests are not run in the Maven and are only run in the CI environment when the pull request is submitted to github. Make sure to watch the [CI results] (https://travis-ci.org/apache/zeppelin/pull_requests) for your pull request.
## Continuous Integration
Zeppelin uses Travis for CI. In the project root there is .travis.yml that configures CI and [publishes CI results] (https://travis-ci.org/apache/zeppelin/builds)
## Run Zeppelin server in development mode
```
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
or use daemon script
```
bin/zeppelin-daemon start
```
Server will be run on http://localhost:8080
## JIRA
Zeppelin manages it's issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
## Where to Start
You can find issues for [beginner](https://issues.apache.org/jira/issues/?jql=(labels%20%3D%20newbie%20or%20labels%3D%20beginner)%20and%20project%3DZeppelin)
## Stay involved
Everyone is welcome to join our mailing list:
* [users@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for usage questions, help, and announcements [ [subscribe](mailto:users-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:users-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) ]
* [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for people who want to contribute code to Zeppelin.[ [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) ]
* [commits@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) is for commit messages and patches to Zeppelin. [ [subscribe](mailto:commits-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:commits-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) ]

10
README.md
View file

@ -3,7 +3,7 @@
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/zeppelin.png?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Contributing:** [Contribution Guide](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md)<br/>
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
**License:** [Apache 2.0](https://github.com/apache/zeppelin/blob/master/LICENSE)
@ -18,7 +18,7 @@ Core feature:
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org)
## Requirements
* Git
* Git
* Java 1.7
* Tested on Mac OSX, Ubuntu 14.X, CentOS 6.X, Windows 7 Pro SP1
* Maven (if you want to build from the source code)
@ -67,7 +67,7 @@ First of all, set your proxy configuration on Maven `settings.xml`.
</settings>
```
Then, run these commands from shell.
Then, run these commands from shell.
```
npm config set proxy http://localhost:3128
npm config set https-proxy http://localhost:3128
@ -87,10 +87,10 @@ git config --global --unset https.proxy
git config --global --unset url."http://".insteadOf
```
_Notes:_
_Notes:_
- If you are behind NTLM proxy you can use [Cntlm Authentication Proxy](http://cntlm.sourceforge.net/).
- Replace `localhost:3128` with the standard pattern `http://user:pwd@host:port`.
#### Install maven
```
wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz

7
bin/common.cmd
View file

@ -81,13 +81,6 @@ if not defined JAVA_OPTS (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_JAVA_OPTS (
set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_MEM (
set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM%
)
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%

9
bin/common.sh
View file

@ -121,15 +121,6 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
# jvm options for interpreter process
if [[ -z "${ZEPPELIN_INTP_JAVA_OPTS}" ]]; then
export ZEPPELIN_INTP_JAVA_OPTS="${ZEPPELIN_JAVA_OPTS}"
fi
if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
export ZEPPELIN_INTP_MEM="${ZEPPELIN_MEM}"
fi
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_INTP_OPTS

3
conf/shiro.ini
View file

@ -28,7 +28,10 @@ user3 = password4, role2
### A sample for configuring Active Directory Realm
#activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
#activeDirectoryRealm.systemUsername = userNameA
#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html
#activeDirectoryRealm.systemPassword = passwordA
#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks
#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
#activeDirectoryRealm.url = ldap://ldap.test.com:389
#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"

4
conf/zeppelin-env.cmd.template
View file

@ -20,8 +20,8 @@ REM set JAVA_HOME=
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options.
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.

4
conf/zeppelin-env.sh.template
View file

@ -20,8 +20,8 @@
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options.
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.

2
conf/zeppelin-site.xml.template
View file

@ -190,7 +190,7 @@
<property>
<name>zeppelin.interpreter.group.order</name>
<value>"spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase</value>
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery</value>
<description></description>
</property>

94
docs/CONTRIBUTING.md
View file

@ -1,94 +0,0 @@
# Contributing to Apache Zeppelin Documentation
## Folder Structure
`docs/` folder is organized as below:
```
docs/
├── _includes/themes/zeppelin
│ ├── _navigation.html
│ └── default.html
├── _layouts
├── _plugins
├── assets/themes/zeppelin -> {ASSET_PATH}
│ ├── bootstrap
│ ├── css
│ ├── img
│ └── js
├── development/ *.md
├── displaysystem/ *.md
├── install/ *.md
├── interpreter/ *.md
├── manual/ *.md
├── quickstart/ *.md
├── rest-api/ *.md
├── security/ *.md
├── storage/ *.md
├── Gemfile
├── Gemfile.lock
├── _config.yml
├── index.md
└── ...
```
- `_navigation.html`: the dropdown menu in navbar
- `default.html` & `_layouts/`: define default HTML layout
- `_plugins/`: custom plugin `*.rb` files can be placed in this folder. See [jekyll/plugins](https://jekyllrb.com/docs/plugins/) for the further information.
- `{ASSET_PATH}/css/style.css`: extra css components can be defined
- `{ASSET_PATH}/img/docs-img/`: image files used for document pages can be placed in this folder
- `{ASSET_PATH}/js/`: extra `.js` files can be placed
- `Gemfile`: defines bundle dependencies. They will be installed by `bundle install`.
- `Gemfile.lock`: when you run `bundle install`, bundler will persist all gems name and their version to this file. For the more details, see [Bundle "The Gemfile Lock"](http://bundler.io/v1.10/man/bundle-install.1.html#THE-GEMFILE-LOCK)
- `documentation_group`: `development/`, `displaysystem/`, `install/`, `interpreter/`...
- `_config.yml`: defines configuration options for docs website. See [jekyll/configuration](https://jekyllrb.com/docs/configuration/) for the other available config variables.
- `index.md`: the main page of `http://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>/`
## Markdown
Zeppelin documentation pages are written with [Markdown](http://daringfireball.net/projects/markdown/). It is possible to use [GitHub flavored syntax](https://help.github.com/categories/writing-on-github/) and intermix plain HTML.
## Front matter
Every page contains [YAML front matter](https://jekyllrb.com/docs/frontmatter/) block in their header. Don't forget to wrap the front matter list with triple-dashed lines(`---`) like below.
The document page should start this triple-dashed lines. Or you will face 404 error, since Jekyll can't find the page.
```
---
layout: page
title: "Apache Zeppelin Tutorial"
description: "This tutorial page contains a short walk-through tutorial that uses Apache Spark backend. Please note that this tutorial is valid for Spark 1.3 and higher."
group: quickstart
---
```
- `layout`: the default layout is `page` which is defined in `_layout/page.html`.
- `title`: the title for the document. Please note that if it needs to include `Zeppelin`, it should be `Apache Zeppelin`, not `Zeppelin`.
- `description`: a short description for the document. One or two sentences would be enough. This description also will be shown as an extract sentence when people search pages.
- `group`: a category of the document page
## Headings
All documents are structured with headings. From these headings, you can automatically generate a **Table of Contents**. There is a simple rule for Zeppelin docs headings.
```
# Level-1 heading <- used only for the main title
## Level-2 heading <- start with this
### Level-3 heading
#### Level-4 heading <- won't be converted in TOC from this level
```
## Table of contents(TOC)
```
<div id="toc"></div>
```
Add this line below `# main title` in order to generate a **Table of Contents**. Headings until `### (Level-3 heading)` are included to TOC.
Default setting options for TOC are definded in [here](https://github.com/apache/zeppelin/blob/master/docs/assets/themes/zeppelin/js/toc.js#L4).
## Adding new pages
If you're going to create new pages, there are some spots you need to add the location of the page.
- **Dropdown menu in navbar**: add your docs location to [_navigation.html](https://github.com/apache/zeppelin/blob/master/docs/_includes/themes/zeppelin/_navigation.html)
- **Main index**: add your docs below [What is the next?](http://zeppelin.apache.org/docs/latest/#what-is-the-next) section in [index.md](https://github.com/apache/zeppelin/blob/master/docs/index.md) with a short description. No need to do this if the page is for **Interpreters**.

8
docs/README.md
View file

@ -1,6 +1,6 @@
# Apache Zeppelin documentation
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest) is also generated from the files found in here.
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest) is also generated from the files found in here.
## Build documentation
Zeppelin is using [Jekyll](https://jekyllrb.com/) which is a static site generator and [Github Pages](https://pages.github.com/) as a site publisher. For the more details, see [help.github.com/articles/about-github-pages-and-jekyll/](https://help.github.com/articles/about-github-pages-and-jekyll/).
@ -19,7 +19,7 @@ bundle install
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
On OS X 10.9, you may need to do
On OS X 10.9, you may need to do
```
xcode-select --install
@ -39,7 +39,7 @@ Using the above command, Jekyll will start a web server at `http://localhost:400
## Contribute to Zeppelin documentation
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://github.com/apache/zeppelin/blob/master/docs/CONTRIBUTING.md).
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://zeppelin.apache.org/contribution/contributions.html).
## For committers only
@ -49,7 +49,7 @@ If you wish to help us and contribute to Zeppelin Documentation, please look at
### Deploy to ASF svnpubsub infra
1. generate static website in `./_site`
```
# go to /docs under Zeppelin source
bundle exec jekyll build --safe

2
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -105,6 +105,8 @@
<li class="title"><span><b>Advanced</b><span></li>
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Contibute</b><span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>

3
docs/assets/themes/zeppelin/css/style.css
View file

@ -438,11 +438,10 @@ a.anchor {
.content table {
display: block;
width: 100%;
overflow: auto;
word-break: normal;
word-break: keep-all;
-webkit-overflow-scrolling: touch;
font-size: 90%;
font-size: 87%;
margin-top: 16px;
margin-bottom: 16px;
}

BIN
docs/assets/themes/zeppelin/img/available_interpreters.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 245 KiB

After

Width:  |  Height:  |  Size: 195 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 76 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 95 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 125 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 212 KiB

6
docs/assets/themes/zeppelin/js/search.js
View file

@ -25,7 +25,7 @@ jQuery(function() {
this.field('group');
});
window.data = $.getJSON('/search_data.json');
window.data = $.getJSON('search_data.json');
window.data.then(function(loaded_data){
$.each(loaded_data, function(index, value){
window.idx.add(
@ -51,6 +51,8 @@ jQuery(function() {
function display_search_results(results) {
var $search_results = $("#search_results");
var zeppelin_version = {{site.ZEPPELIN_VERSION | jsonify}};
var base_url = {{site.JB.BASE_PATH | jsonify}};
var prod_url = {{site.production_url | jsonify}};
window.data.then(function(loaded_data) {
if (results.length) {
@ -59,7 +61,7 @@ jQuery(function() {
results.forEach(function(result) {
var item = loaded_data[result.ref];
var appendString = '<a href="'+item.url+'">'+item.title+'</a><div class="link">'+'https://zeppelin.apache.org/docs/'+zeppelin_version+item.url+'</div><p>'+item.excerpt+'</p><br/>';
var appendString = '<a href="'+base_url+item.url.trim()+'">'+item.title+'</a><div class="link">'+prod_url+base_url+item.url.trim()+'</div><p>'+item.excerpt+'</p><br/>';
$search_results.append(appendString);
});

7
docs/development/writingzeppelininterpreter.md
View file

@ -42,7 +42,7 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
```
{ZEPPELIN_INTERPRETER_DIR}/{YOUR_OWN_INTERPRETER_DIR}/interpreter-setting.json
@ -73,7 +73,7 @@ Here is an example of `interpreter-setting.json` on your own interpreter.
},
{
...
}
}
]
```
@ -207,7 +207,7 @@ Checkout some interpreters released with Zeppelin by default.
We welcome contribution to a new interpreter. Please follow these few steps:
- First, check out the general contribution guide [here](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md).
- First, check out the general contribution guide [here](https://zeppelin.apache.org/contribution/contributions.html).
- Follow the steps in [Make your own Interpreter](#make-your-own-interpreter) section above.
- Add your interpreter as in the [Configure your interpreter](#configure-your-interpreter) section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template).
- Add tests! They are run by [Travis](https://travis-ci.org/apache/zeppelin) for all changes and it is important that they are self-contained.
@ -215,4 +215,3 @@ We welcome contribution to a new interpreter. Please follow these few steps:
- Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`).
- Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE).
- Commit your changes and open a [Pull Request](https://github.com/apache/zeppelin/pulls) on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing.

7
docs/index.md
View file

@ -115,7 +115,7 @@ If you want to learn more about this feature, please visit [this page](./manual/
<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="./assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](./development/howtocontribute.html).
Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](https://zeppelin.apache.org/contribution/contributions.html).
Apache Zeppelin has a very active development community.
Join to our [Mailing list](https://zeppelin.apache.org/community.html) and report issues on [Jira Issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN).
@ -170,14 +170,15 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* Advanced
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-on-yarn-mode)
* [Zeppelin on Spark Cluster Mode (Mesos via Docker)](./install/spark_cluster_mode.html#spark-on-mesos-mode)
* Contribute
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
* [How to contribute (code)](./development/howtocontribute.html)
* [How to contribute (documentation website)](./development/howtocontributewebsite.html)
#### External Resources
#### External Resources
* [Mailing List](https://zeppelin.apache.org/community.html)
* [Apache Zeppelin Wiki](https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home)
* [StackOverflow tag `apache-zeppelin`](http://stackoverflow.com/questions/tagged/apache-zeppelin)

8
docs/install/install.md
View file

@ -195,10 +195,10 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<table class="table-configuration">
<tr>
<th>zepplin-env.sh</th>
<th>zepplin-site.xml</th>
<th>zeppelin-env.sh</th>
<th>zeppelin-site.xml</th>
<th>Default value</th>
<th>Description</th>
<th class="col-md-4">Description</th>
</tr>
<tr>
<td>ZEPPELIN_PORT</td>
@ -354,7 +354,7 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=&lt;accountName&gt;;AccountKey=&lt;accountKey&gt;</code></td>
<td>The Azure storage account connection string<br />i.e. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=&lt;accountName&gt;;<br/>AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>

136
docs/install/spark_cluster_mode.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Apache Zeppelin on Spark cluster mode"
description: ""
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager(Standalone, Hadoop Yarn, Apache Mesos) with Apache Zeppelin using docker scripts."
group: install
---
<!--
@ -56,12 +56,12 @@ spark_standalone bash;
```
### 3. Configure Spark interpreter in Zeppelin
Set Spark master as `spark://localhost:7077` in Zeppelin **Interpreters** setting page.
Set Spark master as `spark://<hostname>:7077` in Zeppelin **Interpreters** setting page.
<img src="../assets/themes/zeppelin/img/docs-img/standalone_conf.png" />
### 4. Run Zeppelin with Spark interpreter
After running single paragraph with Spark interpreter in Zeppelin, browse `https://localhost:8080` and check whether Spark cluster is running well or not.
After running single paragraph with Spark interpreter in Zeppelin, browse `https://<hostname>:8080` and check whether Spark cluster is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/spark_ui.png" />
@ -72,3 +72,133 @@ ps -ef | grep spark
```
## Spark on YARN mode
You can simply set up [Spark on YARN](http://spark.apache.org/docs/latest/running-on-yarn.html) docker environment with below steps.
> **Note :** Since Apache Zeppelin and Spark use same `8080` port for their web UI, you might need to change `zeppelin.server.port` in `conf/zeppelin-site.xml`.
### 1. Build Docker file
You can find docker script files under `scripts/docker/spark-cluster-managers`.
```
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_yarn
docker build -t "spark_yarn" .
```
### 2. Run docker
```
docker run -it \
-p 5000:5000 \
-p 9000:9000 \
-p 9001:9001 \
-p 8088:8088 \
-p 8042:8042 \
-p 8030:8030 \
-p 8031:8031 \
-p 8032:8032 \
-p 8033:8033 \
-p 8080:8080 \
-p 7077:7077 \
-p 8888:8888 \
-p 8081:8081 \
-p 50010:50010 \
-p 50075:50075 \
-p 50020:50020 \
-p 50070:50070 \
--name spark_yarn \
-h sparkmaster \
spark_yarn bash;
```
### 3. Verify running Spark on YARN.
You can simply verify the processes of Spark and YARN are running well in Docker with below command.
```
ps -ef
```
You can also check each application web UI for HDFS on `http://<hostname>:50070/`, YARN on `http://<hostname>:8088/cluster` and Spark on `http://<hostname>:8080/`.
### 4. Configure Spark interpreter in Zeppelin
Set following configurations to `conf/zeppelin-env.sh`.
```
export MASTER=yarn-client
export HADOOP_CONF_DIR=[your_hadoop_conf_path]
export SPARK_HOME=[your_spark_home_path]
```
`HADOOP_CONF_DIR`(Hadoop configuration path) is defined in `/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf`.
Don't forget to set Spark `master` as `yarn-client` in Zeppelin **Interpreters** setting page like below.
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png" />
### 5. Run Zeppelin with Spark interpreter
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/yarn_applications.png" />
## Spark on Mesos mode
You can simply set up [Spark on Mesos](http://spark.apache.org/docs/latest/running-on-mesos.html) docker environment with below steps.
### 1. Build Docker file
```
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_mesos
docker build -t "spark_mesos" .
```
### 2. Run docker
```
docker run --net=host -it \
-p 8080:8080 \
-p 7077:7077 \
-p 8888:8888 \
-p 8081:8081 \
-p 8082:8082 \
-p 5050:5050 \
-p 5051:5051 \
-p 4040:4040 \
-h sparkmaster \
--name spark_mesos \
spark_mesos bash;
```
### 3. Verify running Spark on Mesos.
You can simply verify the processes of Spark and Mesos are running well in Docker with below command.
```
ps -ef
```
You can also check each application web UI for Mesos on `http://<hostname>:5050/cluster` and Spark on `http://<hostname>:8080/`.
### 4. Configure Spark interpreter in Zeppelin
```
export MASTER=mesos://127.0.1.1:5050
export MESOS_NATIVE_JAVA_LIBRARY=[PATH OF libmesos.so]
export SPARK_HOME=[PATH OF SPARK HOME]
```
Don't forget to set Spark `master` as `mesos://127.0.1.1:5050` in Zeppelin **Interpreters** setting page like below.
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png" />
### 5. Run Zeppelin with Spark interpreter
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:5050/#/frameworks` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" />

9
docs/install/upgrade.md
View file

@ -43,4 +43,11 @@ So, copying `notebook` and `conf` directory should be enough.
```
bin/zeppelin-daemon.sh start
```
```
## Migration Guide
### Upgrading from Zeppelin 0.6 to 0.7
- From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly.
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.

79
docs/rest-api/rest-notebook.md
View file

@ -298,7 +298,10 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
<col width="200">
<tr>
<td>Description</td>
<td>This ```POST``` method runs all paragraphs in the given notebook id.
<td>
This ```POST``` method runs all paragraphs in the given notebook id. <br />
If you can not find Notebook id 404 returns.
If there is a problem with the interpreter returns a 412 error.
</td>
</tr>
<tr>
@ -311,12 +314,29 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
<td> 404 or 412</td>
</tr>
<tr>
<td> sample JSON response </td>
<td><pre>{"status": "OK"}</pre></td>
</tr>
<tr>
<td> sample JSON error response </td>
<td>
<pre>
{
"status": "NOT_FOUND",
"message": "note not found."
}
</pre><br />
<pre>
{
"status": "PRECONDITION_FAILED",
"message": "paragraph_1469771130099_-278315611 Not selected or Invalid Interpreter bind"
}
</pre>
</td>
</tr>
</table>
<br/>
@ -430,12 +450,12 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</table>
<br/>
### Run a paragraph
### Run a paragraph asynchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
<td>This ```POST``` method runs the paragraph asynchronously by given notebook and paragraph id. This API always return SUCCESS even if the execution of the paragraph fails later because the API is asynchronous
</td>
</tr>
<tr>
@ -467,6 +487,56 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</table>
<br/>
### Run a paragraph synchronously
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td> This ```POST``` method runs the paragraph synchronously by given notebook and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON input (optional, only needed when if you want to update dynamic form's value) </td>
<td><pre>
{
"name": "name of new notebook",
"params": {
"formLabel1": "value1",
"formLabel2": "value2"
}
}</pre></td>
</tr>
<tr>
<td> sample JSON response </td>
<td><pre>{"status": "OK"}</pre></td>
</tr>
<tr>
<td> sample JSON error </td>
<td><pre>
{
"status": "INTERNAL\_SERVER\_ERROR",
"body": {
"code": "ERROR",
"type": "TEXT",
"msg": "bash: -c: line 0: unexpected EOF while looking for matching ``'\nbash: -c: line 1: syntax error: unexpected end of file\nExitValue: 2"
}
}</pre></td>
</tr>
</table>
<br/>
### Stop a paragraph
<table class="table-configuration">
@ -902,4 +972,3 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</tr>
</table>

4
docs/screenshots.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Screenshots"
description: ""
title:
description:
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");

30
docs/security/shiroauthentication.md
View file

@ -112,10 +112,36 @@ To learn more about Apache Shiro Realm, please check [this documentation](http:/
We also provide community custom Realms.
### Active Directory
TBD
```
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
activeDirectoryRealm.systemUsername = userNameA
activeDirectoryRealm.systemPassword = passwordA
activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks
activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
activeDirectoryRealm.url = ldap://ldap.test.com:389
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
activeDirectoryRealm.authorizationCachingEnabled = false
```
Also instead of specifying systemPassword in clear text in shiro.ini administrator can choose to specify the same in "hadoop credential".
Create a keystore file using the hadoop credential commandline, for this the hadoop commons should be in the classpath
`hadoop credential create activeDirectoryRealm.systempassword -provider jceks://file/user/zeppelin/conf/zeppelin.jceks`
Change the following values in the Shiro.ini file, and uncomment the line:
`activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks`
### LDAP
TBD
```
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
# search base for ldap groups (only relevant for LdapGroupRealm):
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
ldapRealm.contextFactory.url = ldap://ldap.test.com:389
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
```
### ZeppelinHub
[ZeppelinHub](https://www.zeppelinhub.com) is a service that synchronize your Apache Zeppelin notebooks and enables you to collaborate easily.

25
flink/pom.xml
View file

@ -120,21 +120,18 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -146,24 +143,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<!-- Scala Compiler -->
<plugin>
@ -319,7 +298,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
@ -339,10 +317,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>

2
ignite/pom.xml
View file

@ -116,8 +116,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>

96
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -15,7 +15,8 @@
package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
@ -26,7 +27,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -167,7 +168,7 @@ public class JDBCInterpreter extends Interpreter {
logger.debug("propertiesMap: {}", propertiesMap);
if (!StringUtils.isAnyEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
if (!StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
JDBCSecurityImpl.createSecureConfiguration(property);
}
for (String propertyKey : propertiesMap.keySet()) {
@ -214,49 +215,52 @@ public class JDBCInterpreter extends Interpreter {
Class.forName(properties.getProperty(DRIVER_KEY));
final String url = properties.getProperty(URL_KEY);
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = DriverManager.getConnection(url, properties);
} else {
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = DriverManager.getConnection(url + ";hive.server2.proxy.user=" + user,
properties);
if (StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
connection = DriverManager.getConnection(url, properties);
} else {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = DriverManager.getConnection(url, properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return DriverManager.getConnection(url, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = DriverManager.getConnection(url + ";hive.server2.proxy.user=" + user,
properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return DriverManager.getConnection(url, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
}
}
}
break;
break;
default:
connection = DriverManager.getConnection(url, properties);
default:
connection = DriverManager.getConnection(url, properties);
}
}
}
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
return connection;
@ -418,11 +422,11 @@ public class JDBCInterpreter extends Interpreter {
} catch (Exception e) {
logger.error("Cannot run " + sql, e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getClass().toString()).append("\n");
stringBuilder.append(StringUtils.join(e.getStackTrace(), "\n"));
return new InterpreterResult(Code.ERROR, stringBuilder.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
String errorMsg = new String(baos.toByteArray(), StandardCharsets.UTF_8);
return new InterpreterResult(Code.ERROR, errorMsg);
}
}

1
jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java
View file

@ -18,7 +18,6 @@ package org.apache.zeppelin.jdbc.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.jdbc.SqlCompleter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

255
notebook/r/note.json → notebook/2BWJFTXKJ/note.json
View file

File diff suppressed because one or more lines are too long

281
pom.xml
View file

@ -211,11 +211,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
@ -389,7 +384,7 @@
</plugin>
<!--TODO(alex): make part of the build and reconcile conflicts
<plugin>
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.4</version>
@ -398,12 +393,12 @@
<id>default</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict>
</configuration>
</plugin>
-->
@ -411,129 +406,6 @@
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>CONTRIBUTING.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>docs/CONTRIBUTING.md</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@ -659,7 +531,7 @@
<profile>
<id>scala-2.10</id>
<activation>
<activeByDefault>true</activeByDefault>
<property><name>!scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.10.5</scala.version>
@ -669,6 +541,9 @@
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.11.7</scala.version>
<scala.binary.version>2.11</scala.binary.version>
@ -816,6 +691,142 @@
</plugins>
</build>
</profile>
<profile>
<id>rat</id>
<activation>
<property><name>!skipRat</name></property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>reports/**</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.rat-excludes/</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
<exclude>**/*.avro</exclude>
<exclude>**/*.log</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/logs/**</exclude>
<exclude>**/run/**</exclude>
<exclude>**/interpreter/**</exclude>
<exclude>**/local-repo/**</exclude>
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>**/licenses/**</exclude>
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/credentials.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
<!-- bundled from jekyll-bootstrap -->
<exclude>docs/404.html</exclude>
<exclude>docs/_config.yml</exclude>
<exclude>docs/_includes/JB/**</exclude>
<exclude>docs/_layouts/**</exclude>
<exclude>docs/_plugins/**</exclude>
<exclude>docs/atom.xml</exclude>
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
<exclude>docs/Rakefile</exclude>
<exclude>docs/rss.xml</exclude>
<exclude>docs/sitemap.txt</exclude>
<exclude>docs/search_data.json</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<!-- bundled from anchor -->
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
<!-- bundled from toc -->
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
<!-- bundled from lunrjs -->
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
<!-- bundled from jekyll -->
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
<!-- docs (website) build target dir -->
<exclude>docs/_site/**</exclude>
<exclude>docs/Gemfile.lock</exclude>
<exclude>**/horizontalbar_mockdata.txt</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<exclude>r/lib/**</exclude>
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>
<executions>
<execution>
<id>verify.rat</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

13
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -150,8 +150,17 @@ public class PythonInterpreter extends Interpreter {
* @return true if syntax error or exception has happened
*/
private boolean pythonErrorIn(String output) {
Matcher errorMatcher = errorInLastLine.matcher(output);
return errorMatcher.find();
boolean isError = false;
String[] outputMultiline = output.split("\n");
Matcher errorMatcher;
for (String row : outputMultiline) {
errorMatcher = errorInLastLine.matcher(row);
if (errorMatcher.find() == true) {
isError = true;
break;
}
}
return isError;
}
@Override

20
python/src/main/resources/bootstrap.py
View file

@ -117,7 +117,6 @@ class PyZeppelinContext(object):
def __init__(self):
self.max_result = 1000
self.py3 = bool(sys.version_info >= (3,))
def input(self, name, defaultValue=""):
print(self.errorMsg)
@ -165,28 +164,27 @@ class PyZeppelinContext(object):
#)
body_buf.close(); header_buf.close()
def show_matplotlib(self, p, width="100%", height="100%",
fmt='png', **kwargs):
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
**kwargs):
"""Matplotlib show function
"""
if fmt == 'png':
if fmt == "png":
img = BytesIO()
p.savefig(img, format=fmt)
html = "%html <img src={img} width={width}, height={height}>"
img_str = b"data:image/png;base64,"
img_str += base64.b64encode(img.getvalue().strip())
# Need to do this for python3 compatibility
if self.py3:
img_str = img_str.decode('ascii')
elif fmt == 'svg':
img_tag = "<img src={img} style='width={width};height:{height}'>"
# Decoding is necessary for Python 3 compability
img_str = img_str.decode("ascii")
img_str = img_tag.format(img=img_str, width=width, height=height)
elif fmt == "svg":
img = StringIO()
p.savefig(img, format=fmt)
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
img_str = img.getvalue()
else:
raise ValueError("fmt must be 'png' or 'svg'")
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
print(html.format(width=width, height=height, img=img_str))
img.close()

21
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -218,4 +218,25 @@ public class PythonInterpreterTest {
}
}
@Test
public void checkMultiRowErrorFails() {
PythonInterpreter pythonInterpreter = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties()
);
pythonInterpreter.open();
String codeRaiseException = "raise Exception(\"test exception\")";
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
assertNotNull("Interpreter result for raise exception is Null", ret);
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().length() > 0);
assertNotNull("Interpreter result for text is Null", ret);
String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
ret = pythonInterpreter.interpret(codePrintText, null);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().length() > 0);
}
}

20
r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.rinterpreter
import java.io.{BufferedInputStream, File, FileInputStream}
import java.nio.file.{Files, Paths}
import java.util._
@ -110,10 +111,10 @@ object RInterpreter {
// These are the additional properties we need on top of the ones provided by the spark interpreters
lazy val props: Map[String, InterpreterProperty] = new InterpreterPropertyBuilder()
.add("rhadoop.cmd", SparkInterpreter.getSystemDefault("HADOOP_CMD", "rhadoop.cmd", ""), "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("HADOOP_STREAMING", "rhadooop.streamingjar", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", SparkInterpreter.getSystemDefault("RSCALA_DEBUG", "rscala.debug","false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", SparkInterpreter.getSystemDefault("RSCALA_TIMEOUT", "rscala.timeout","60"), "Timeout for rScala") // TEST: Implemented but not tested
.add("rhadoop.cmd", "HADOOP_CMD", "rhadoop.cmd", "", "Usually /usr/bin/hadoop")
.add("rhadooop.streamingjar", "HADOOP_STREAMING", "rhadooop.streamingjar", "", "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
.add("rscala.debug", "RSCALA_DEBUG", "rscala.debug","false", "Whether to turn on rScala debugging") // TEST: Implemented but not tested
.add("rscala.timeout", "RSCALA_TIMEOUT", "rscala.timeout","60", "Timeout for rScala") // TEST: Implemented but not tested
.build
def getProps() = {
@ -141,8 +142,15 @@ object RInterpreter {
}
def dataURI(file : String, mime : String) : String = {
val data: String = Source.fromFile(file).getLines().mkString("\n")
s"""data:${mime};base64,""" + StringUtils.newStringUtf8(Base64.encodeBase64(data.getBytes(), false))
val fp = new File(file)
val fdata = new Array[Byte](fp.length().toInt)
val fin = new BufferedInputStream(new FileInputStream(fp))
try {
fin.read(fdata)
} finally {
fin.close()
}
s"""data:${mime};base64,""" + StringUtils.newStringUtf8(Base64.encodeBase64(fdata, false))
}
// The purpose here is to deal with knitr producing HTML with script and css tags outside the <body>

63
scripts/docker/spark-cluster-managers/spark_mesos/Dockerfile Normal file
View file

@ -0,0 +1,63 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV HADOOP_PROFILE 2.3
ENV HADOOP_VERSION 2.3.0
# Update the image with the latest packages
RUN yum update -y; yum clean all
# Get utils
RUN yum install -y \
wget \
tar \
curl \
svn \
cyrus-sasl-md5 \
libevent2-devel \
&& \
yum clean all
# Remove old jdk
RUN yum remove java; yum remove jdk
# install jdk7
RUN yum install -y java-1.7.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java
ENV PATH $PATH:$JAVA_HOME/bin
# install spark
RUN curl -s http://www.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
# update boot script
COPY entrypoint.sh /etc/entrypoint.sh
RUN chown root.root /etc/entrypoint.sh
RUN chmod 700 /etc/entrypoint.sh
# install mesos
RUN wget http://repos.mesosphere.com/el/6/x86_64/RPMS/mesos-1.0.0-2.0.89.centos65.x86_64.rpm
RUN rpm -Uvh mesos-1.0.0-2.0.89.centos65.x86_64.rpm
#spark
EXPOSE 8080 7077 7072 8081 8082
#mesos
EXPOSE 5050 5051
ENTRYPOINT ["/etc/entrypoint.sh"]

48
scripts/docker/spark-cluster-managers/spark_mesos/entrypoint.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
export SPARK_HOME=/usr/local/spark/
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=8080
export SPARK_WORKER_PORT=8888
export SPARK_WORKER_WEBUI_PORT=8081
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server/
# spark configuration
cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so" >> $SPARK_HOME/conf/spark-env.sh
cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf
echo "spark.master mesos://`hostname`:5050" >> $SPARK_HOME/conf/spark-defaults.conf
echo "spark.mesos.executor.home /usr/local/spark" >> $SPARK_HOME/conf/spark-defaults.conf
# run spark
cd $SPARK_HOME/sbin
./start-master.sh
./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
# start mesos
mesos-master --ip=0.0.0.0 --work_dir=/var/lib/mesos &> /var/log/mesos_master.log &
mesos-slave --master=0.0.0.0:5050 --work_dir=/var/lib/mesos --launcher=posix &> /var/log/mesos_slave.log &
CMD=${1:-"exit 0"}
if [[ "$CMD" == "-d" ]];
then
service sshd stop
/usr/sbin/sshd -D -d
else
/bin/bash -c "$*"
fi

1
scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile
View file

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM centos:centos6
MAINTAINER hsshim@nflabs.com
ENV SPARK_PROFILE 1.6
ENV SPARK_VERSION 1.6.2

107
scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile Normal file
View file

@ -0,0 +1,107 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV HADOOP_PROFILE 2.7
ENV HADOOP_VERSION 2.7.0
# Update the image with the latest packages
RUN yum update -y; yum clean all
# Get utils
RUN yum install -y \
wget \
tar \
curl \
&& \
yum clean all
# Remove old jdk
RUN yum remove java; yum remove jdk
# install jdk7
RUN yum install -y java-1.7.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java
ENV PATH $PATH:$JAVA_HOME/bin
# install hadoop
RUN yum install -y curl which tar sudo openssh-server openssh-clients rsync
# hadoop
RUN curl -s https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s ./hadoop-$HADOOP_VERSION hadoop
ENV HADOOP_PREFIX /usr/local/hadoop
ENV HADOOP_COMMON_HOME /usr/local/hadoop
ENV HADOOP_HDFS_HOME /usr/local/hadoop
ENV HADOOP_MAPRED_HOME /usr/local/hadoop
ENV HADOOP_YARN_HOME /usr/local/hadoop
ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/jre-1.7.0-openjdk.x86_64\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
RUN mkdir $HADOOP_PREFIX/input
RUN cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
# hadoop configurations
ADD hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml
ADD hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
ADD hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
ADD hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
RUN mkdir /data/
RUN chmod 777 /data/
RUN $HADOOP_PREFIX/bin/hdfs namenode -format
RUN rm /usr/local/hadoop/lib/native/*
RUN curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-$HADOOP_VERSION.tar|tar -x -C /usr/local/hadoop/lib/native/
# install spark
RUN curl -s http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
ENV SPARK_HOME /usr/local/spark
ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin
# passwordless ssh
RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa
RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
ADD ssh_config /root/.ssh/config
RUN chmod 600 /root/.ssh/config
RUN chown root:root /root/.ssh/config
RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh
# update boot script
COPY entrypoint.sh /etc/entrypoint.sh
RUN chown root.root /etc/entrypoint.sh
RUN chmod 700 /etc/entrypoint.sh
# Hdfs ports
EXPOSE 50010 50020 50070 50075 50090
# Mapred ports
EXPOSE 9000 9001
#Yarn ports
EXPOSE 8030 8031 8032 8033 8040 8042 8088
#spark
EXPOSE 8080 7077 8888 8081
ENTRYPOINT ["/etc/entrypoint.sh"]

60
scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh Executable file
View file

@ -0,0 +1,60 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
echo 'hadoop' |passwd root --stdin
: ${HADOOP_PREFIX:=/usr/local/hadoop}
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
rm /tmp/*.pid
# installing libraries if any - (resource urls added comma separated to the ACP system variable)
cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
# start hadoop
service sshd start
$HADOOP_PREFIX/sbin/start-dfs.sh
$HADOOP_PREFIX/sbin/start-yarn.sh
$HADOOP_PREFIX/bin/hdfs dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE/lib /spark
# start spark
export SPARK_MASTER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
-Dspark.broadcast.port=7003 -Dspark.replClassServer.port=7004
-Dspark.blockManager.port=7005 -Dspark.executor.port=7006
-Dspark.ui.port=4040 -Dspark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory"
export SPARK_WORKER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
-Dspark.broadcast.port=7003 -Dspark.replClassServer.port=7004
-Dspark.blockManager.port=7005 -Dspark.executor.port=7006
-Dspark.ui.port=4040 -Dspark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory"
export SPARK_MASTER_PORT=7077
cd /usr/local/spark/sbin
./start-master.sh
./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
CMD=${1:-"exit 0"}
if [[ "$CMD" == "-d" ]];
then
service sshd stop
/usr/sbin/sshd -D -d
else
/bin/bash -c "$*"
fi

22
scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml Normal file
View file

@ -0,0 +1,22 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://0.0.0.0:9000</value>
</property>
</configuration>

78
scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/hdfs-site.xml Normal file
View file

@ -0,0 +1,78 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.data.dir</name>
<value>/data/hdfs</value>
<final>true</final>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>true</value>
<description>Whether clients should use datanode hostnames when
connecting to datanodes.
</description>
</property>
<property>
<name>dfs.datanode.use.datanode.hostname</name>
<value>true</value>
<description>Whether datanodes should use datanode hostnames when
connecting to other datanodes for data transfer.
</description>
</property>
<property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:50010</value>
<description>
The address where the datanode server will listen to.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>0.0.0.0:50075</value>
<description>
The datanode http server address and port.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.datanode.ipc.address</name>
<value>0.0.0.0:50020</value>
<description>
The datanode ipc server address and port.
If the port is 0 then the server will start on a free port.
</description>
</property>
</configuration>

22
scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/mapred-site.xml Normal file
View file

@ -0,0 +1,22 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

42
scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/yarn-site.xml Normal file
View file

@ -0,0 +1,42 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>0.0.0.0:8030</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>0.0.0.0:8032</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>0.0.0.0:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>0.0.0.0:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>0.0.0.0:8033</value>
</property>
<property>
<name>yarn.application.classpath</name>
<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*, /usr/local/hadoop/share/spark/*</value>
</property>
</configuration>

18
scripts/docker/spark-cluster-managers/spark_yarn_cluster/ssh_config Normal file
View file

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Host *
UserKnownHostsFile /dev/null
StrictHostKeyChecking no
LogLevel quiet

20
spark-dependencies/pom.xml
View file

@ -959,26 +959,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

344
spark/pom.xml
View file

@ -301,27 +301,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/interpreter-setting.json</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
@ -482,27 +461,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.1</id>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.1.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<spark.version>1.1.1</spark.version>
<akka.version>2.2.3-shaded-protobuf</akka.version>
</properties>
</profile>
<profile>
<id>spark-1.2</id>
<dependencies>
@ -512,26 +470,6 @@
</properties>
</profile>
<profile>
<id>cassandra-spark-1.2</id>
<properties>
<spark.version>1.2.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.2.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.3</id>
@ -544,27 +482,6 @@
</profile>
<profile>
<id>cassandra-spark-1.3</id>
<properties>
<spark.version>1.3.0</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.3.1</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.4</id>
<properties>
@ -575,27 +492,6 @@
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.4</id>
<properties>
<spark.version>1.4.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.4.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-1.5</id>
<properties>
@ -604,34 +500,6 @@
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
</properties>
<dependencies>
</dependencies>
</profile>
<profile>
<id>cassandra-spark-1.5</id>
<properties>
<spark.version>1.5.1</spark.version>
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
<guava.version>16.0.1</guava.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
@ -732,218 +600,6 @@
</properties>
</profile>
<profile>
<id>mapr3</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>1.0.3-mapr-3.0.3</hadoop.version>
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr40</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.4.1-mapr-1503</hadoop.version>
<yarn.version>2.4.1-mapr-1503</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr41</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.5.1-mapr-1503</hadoop.version>
<yarn.version>2.5.1-mapr-1503</yarn.version>
<jets3t.version>0.7.1</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr50</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1506</hadoop.version>
<yarn.version>2.7.0-mapr-1506</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>mapr51</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1602</hadoop.version>
<yarn.version>2.7.0-mapr-1602</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>
</profile>
<profile>
<id>yarn</id>
<properties>
<yarn.version>${hadoop.version}</yarn.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
</dependencies>
</profile>
<!-- include sparkr in the build -->
<profile>
<id>sparkr</id>

38
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Joiner;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
@ -48,6 +49,7 @@ import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
@ -300,7 +302,9 @@ public class SparkInterpreter extends Interpreter {
String execUri = System.getenv("SPARK_EXECUTOR_URI");
conf.setAppName(getProperty("spark.app.name"));
conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath());
if (outputDir != null) {
conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath());
}
if (execUri != null) {
conf.set("spark.executor.uri", execUri);
@ -444,10 +448,11 @@ public class SparkInterpreter extends Interpreter {
}
private void setupConfForPySpark(SparkConf conf) {
String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null);
String pysparkBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue();
File pysparkPath;
if (null == pysparkBasePath) {
pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
pysparkBasePath =
new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue();
pysparkPath = new File(pysparkBasePath,
"interpreter" + File.separator + "spark" + File.separator + "pyspark");
} else {
@ -497,27 +502,6 @@ public class SparkInterpreter extends Interpreter {
return null != System.getenv("SPARK_SUBMIT");
}
public static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
return envValue;
}
}
if (propertyName != null && !propertyName.isEmpty()) {
String propValue = System.getProperty(propertyName);
if (propValue != null) {
return propValue;
}
}
return defaultValue;
}
public boolean printREPLOutput() {
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput"));
}
@ -593,7 +577,11 @@ public class SparkInterpreter extends Interpreter {
argList.add("-Yrepl-class-based");
argList.add("-Yrepl-outdir");
argList.add(outputDir.getAbsolutePath());
if (conf.contains("spark.jars")) {
String jars = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
argList.add("-classpath");
argList.add(jars);
}
scala.collection.immutable.List<String> list =
JavaConversions.asScalaBuffer(argList).toList();

2
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -78,7 +78,7 @@ public class SparkRInterpreter extends Interpreter {
ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession());
}
ZeppelinRContext.setSqlContext(sparkInterpreter.getSQLContext());
ZeppelinRContext.setZepplinContext(sparkInterpreter.getZeppelinContext());
ZeppelinRContext.setZeppelinContext(sparkInterpreter.getZeppelinContext());
zeppelinR = new ZeppelinR(rCmdPath, sparkRLibPath, port, sparkVersion);
try {

2
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java
View file

@ -33,7 +33,7 @@ public class ZeppelinRContext {
ZeppelinRContext.sparkContext = sparkContext;
}
public static void setZepplinContext(ZeppelinContext zeppelinContext) {
public static void setZeppelinContext(ZeppelinContext zeppelinContext) {
ZeppelinRContext.zeppelinContext = zeppelinContext;
}

23
zeppelin-display/pom.xml
View file

@ -81,6 +81,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -94,11 +95,15 @@
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencies>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
@ -106,24 +111,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>

3
zeppelin-distribution/pom.xml
View file

@ -114,6 +114,9 @@
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencyManagement>
<dependencies>
<dependency>

10
zeppelin-examples/zeppelin-example-horizontalbar/pom.xml
View file

@ -105,16 +105,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/horizontalbar_mockdata.txt</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

18
zeppelin-interpreter/pom.xml
View file

@ -217,24 +217,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>*.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -345,10 +345,14 @@ public abstract class Interpreter {
}
public static void register(RegisteredInterpreter registeredInterpreter) {
// TODO(jongyoul): Error should occur when two same interpreter key with different settings
String interpreterKey = registeredInterpreter.getInterpreterKey();
if (!registeredInterpreters.containsKey(interpreterKey)) {
registeredInterpreters.put(interpreterKey, registeredInterpreter);
} else {
RegisteredInterpreter existInterpreter = registeredInterpreters.get(interpreterKey);
if (!existInterpreter.getProperties().equals(registeredInterpreter.getProperties())) {
logger.error("exist registeredInterpreter with the same key but has different settings.");
}
}
}

13
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
View file

@ -70,8 +70,15 @@ public class InterpreterProperty {
this.description = description;
}
public int hashCode() {
return this.toString().hashCode();
}
public boolean equals(Object o) {
return this.toString().equals(o.toString());
}
public String getValue() {
//TODO(jongyoul): Remove SparkInterpreter's getSystemDefault method
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
@ -90,7 +97,7 @@ public class InterpreterProperty {
@Override
public String toString() {
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s", envName,
propertyName, defaultValue, description);
return String.format("{envName=%s, propertyName=%s, defaultValue=%s, description=%20s}",
envName, propertyName, defaultValue, description);
}
}

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java
View file

@ -31,6 +31,13 @@ public class InterpreterPropertyBuilder {
return this;
}
public InterpreterPropertyBuilder add(String name, String envName, String propertyName,
String defaultValue, String description){
properties.put(name,
new InterpreterProperty(envName, propertyName, defaultValue, description));
return this;
}
public Map<String, InterpreterProperty> build(){
return properties;
}

48
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
/**
* This element stores the buffered
* append-data of paragraph's output.
*/
public class AppendOutputBuffer {
private String noteId;
private String paragraphId;
private String data;
public AppendOutputBuffer(String noteId, String paragraphId, String data) {
this.noteId = noteId;
this.paragraphId = paragraphId;
this.data = data;
}
public String getNoteId() {
return noteId;
}
public String getParagraphId() {
return paragraphId;
}
public String getData() {
return data;
}
}

118
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java Normal file
View file

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This thread sends paragraph's append-data
* periodically, rather than continously, with
* a period of BUFFER_TIME_MS. It handles append-data
* for all paragraphs across all notebooks.
*/
public class AppendOutputRunner implements Runnable {
private static final Logger logger =
LoggerFactory.getLogger(AppendOutputRunner.class);
public static final Long BUFFER_TIME_MS = new Long(100);
private static final Long SAFE_PROCESSING_TIME = new Long(10);
private static final Long SAFE_PROCESSING_STRING_SIZE = new Long(100000);
private final BlockingQueue<AppendOutputBuffer> queue = new LinkedBlockingQueue<>();
private final RemoteInterpreterProcessListener listener;
public AppendOutputRunner(RemoteInterpreterProcessListener listener) {
this.listener = listener;
}
@Override
public void run() {
Map<String, Map<String, StringBuilder> > noteMap = new HashMap<>();
List<AppendOutputBuffer> list = new LinkedList<>();
/* "drainTo" method does not wait for any element
* to be present in the queue, and thus this loop would
* continuosly run (with period of BUFFER_TIME_MS). "take()" method
* waits for the queue to become non-empty and then removes
* one element from it. Rest elements from queue (if present) are
* removed using "drainTo" method. Thus we save on some un-necessary
* cpu-cycles.
*/
try {
list.add(queue.take());
} catch (InterruptedException e) {
logger.error("Wait for OutputBuffer queue interrupted: " + e.getMessage());
}
Long processingStartTime = System.currentTimeMillis();
queue.drainTo(list);
for (AppendOutputBuffer buffer: list) {
String noteId = buffer.getNoteId();
String paragraphId = buffer.getParagraphId();
Map<String, StringBuilder> paragraphMap = (noteMap.containsKey(noteId)) ?
noteMap.get(noteId) : new HashMap<String, StringBuilder>();
StringBuilder builder = paragraphMap.containsKey(paragraphId) ?
paragraphMap.get(paragraphId) : new StringBuilder();
builder.append(buffer.getData());
paragraphMap.put(paragraphId, builder);
noteMap.put(noteId, paragraphMap);
}
Long processingTime = System.currentTimeMillis() - processingStartTime;
if (processingTime > SAFE_PROCESSING_TIME) {
logger.warn("Processing time for buffered append-output is high: " +
processingTime + " milliseconds.");
} else {
logger.debug("Processing time for append-output took "
+ processingTime + " milliseconds");
}
Long sizeProcessed = new Long(0);
for (String noteId: noteMap.keySet()) {
for (String paragraphId: noteMap.get(noteId).keySet()) {
String data = noteMap.get(noteId).get(paragraphId).toString();
sizeProcessed += data.length();
listener.onOutputAppend(noteId, paragraphId, data);
}
}
if (sizeProcessed > SAFE_PROCESSING_STRING_SIZE) {
logger.warn("Processing size for buffered append-output is high: " +
sizeProcessed + " characters.");
} else {
logger.debug("Processing size for append-output is " +
sizeProcessed + " characters");
}
}
public void appendBuffer(String noteId, String paragraphId, String outputToAppend) {
queue.offer(new AppendOutputBuffer(noteId, paragraphId, outputToAppend));
}
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -295,7 +295,9 @@ public class RemoteInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;

14
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -39,12 +39,18 @@ import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
/**
* Processes message from RemoteInterpreter process
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
private static final ScheduledExecutorService appendService =
Executors.newSingleThreadScheduledExecutor();
private final RemoteInterpreterProcessListener listener;
private final ApplicationEventListener appListener;
@ -72,6 +78,9 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
while (!shutdown) {
// wait and retry
@ -157,7 +166,7 @@ public class RemoteInterpreterEventPoller extends Thread {
String appId = outputAppend.get("appId");
if (appId == null) {
listener.onOutputAppend(noteId, paragraphId, outputToAppend);
runner.appendBuffer(noteId, paragraphId, outputToAppend);
} else {
appListener.onOutputAppend(noteId, paragraphId, appId, outputToAppend);
}
@ -192,6 +201,9 @@ public class RemoteInterpreterEventPoller extends Thread {
logger.error("Can't handle event " + event, e);
}
}
if (appendFuture != null) {
appendFuture.cancel(true);
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -34,6 +34,7 @@ import org.apache.zeppelin.display.*;
import org.apache.zeppelin.helium.*;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer;
import org.apache.zeppelin.interpreter.thrift.*;
import org.apache.zeppelin.resource.*;
import org.apache.zeppelin.scheduler.Job;
@ -133,7 +134,11 @@ public class RemoteInterpreterServer
public static void main(String[] args)
throws TTransportException, InterruptedException {
int port = Integer.parseInt(args[0]);
int port = ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT;
if (args.length > 0) {
port = Integer.parseInt(args[0]);
}
RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(port);
remoteInterpreterServer.start();
remoteInterpreterServer.join();
@ -280,7 +285,9 @@ public class RemoteInterpreterServer
@Override
public RemoteInterpreterResult interpret(String noteId, String className, String st,
RemoteInterpreterContext interpreterContext) throws TException {
logger.debug("st: {}", st);
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
Interpreter intp = getInterpreter(noteId, className);
InterpreterContext context = convert(interpreterContext);

235
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java Normal file
View file

@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.After;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;
private static final int NUM_CLUBBED_EVENTS = 100;
private static final ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
private static ScheduledFuture<?> future = null;
/* It is being accessed by multiple threads.
* While loop for 'loopForBufferCompletion' could
* run for-ever.
*/
private volatile static int numInvocations = 0;
@After
public void afterEach() {
if (future != null) {
future.cancel(true);
}
}
@Test
public void testSingleEvent() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String[][] buffer = {{"note", "para", "data\n"}};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend("note", "para", "data\n");
}
@Test
public void testMultipleEventsOfSameParagraph() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String para1 = "para1";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para1, "data2\n"},
{note1, para1, "data3\n"}
};
loopForCompletingEvents(listener, 1, buffer);
verify(listener, times(1)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\ndata2\ndata3\n");
}
@Test
public void testMultipleEventsOfDifferentParagraphs() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
String note1 = "note1";
String note2 = "note2";
String para1 = "para1";
String para2 = "para2";
String[][] buffer = {
{note1, para1, "data1\n"},
{note1, para2, "data2\n"},
{note2, para1, "data3\n"},
{note2, para2, "data4\n"}
};
loopForCompletingEvents(listener, 4, buffer);
verify(listener, times(4)).onOutputAppend(any(String.class), any(String.class), any(String.class));
verify(listener, times(1)).onOutputAppend(note1, para1, "data1\n");
verify(listener, times(1)).onOutputAppend(note1, para2, "data2\n");
verify(listener, times(1)).onOutputAppend(note2, para1, "data3\n");
verify(listener, times(1)).onOutputAppend(note2, para2, "data4\n");
}
@Test
public void testClubbedData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
Thread thread = new Thread(new BombardEvents(runner));
thread.start();
thread.join();
Thread.sleep(1000);
/* NUM_CLUBBED_EVENTS is a heuristic number.
* It has been observed that for 10,000 continuos event
* calls, 30-40 Web-socket calls are made. Keeping
* the unit-test to a pessimistic 100 web-socket calls.
*/
verify(listener, atMost(NUM_CLUBBED_EVENTS)).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
@Test
public void testWarnLoggerForLargeData() throws InterruptedException {
RemoteInterpreterProcessListener listener = mock(RemoteInterpreterProcessListener.class);
AppendOutputRunner runner = new AppendOutputRunner(listener);
String data = "data\n";
int numEvents = 100000;
for (int i=0; i<numEvents; i++) {
runner.appendBuffer("noteId", "paraId", data);
}
TestAppender appender = new TestAppender();
Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
Logger.getLogger(RemoteInterpreterEventPoller.class);
runner.run();
List<LoggingEvent> log;
int warnLogCounter;
LoggingEvent sizeWarnLogEntry = null;
do {
warnLogCounter = 0;
log = appender.getLog();
for (LoggingEvent logEntry: log) {
if (Level.WARN.equals(logEntry.getLevel())) {
sizeWarnLogEntry = logEntry;
warnLogCounter += 1;
}
}
} while(warnLogCounter != 2);
String loggerString = "Processing size for buffered append-output is high: " +
(data.length() * numEvents) + " characters.";
assertTrue(loggerString.equals(sizeWarnLogEntry.getMessage()));
}
private class BombardEvents implements Runnable {
private final AppendOutputRunner runner;
private BombardEvents(AppendOutputRunner runner) {
this.runner = runner;
}
@Override
public void run() {
String noteId = "noteId";
String paraId = "paraId";
for (int i=0; i<NUM_EVENTS; i++) {
runner.appendBuffer(noteId, paraId, "data\n");
}
}
}
private class TestAppender extends AppenderSkeleton {
private final List<LoggingEvent> log = new ArrayList<>();
@Override
public boolean requiresLayout() {
return false;
}
@Override
protected void append(final LoggingEvent loggingEvent) {
log.add(loggingEvent);
}
@Override
public void close() {
}
public List<LoggingEvent> getLog() {
return new ArrayList<>(log);
}
}
private void prepareInvocationCounts(RemoteInterpreterProcessListener listener) {
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
numInvocations += 1;
return null;
}
}).when(listener).onOutputAppend(any(String.class), any(String.class), any(String.class));
}
private void loopForCompletingEvents(RemoteInterpreterProcessListener listener,
int numTimes, String[][] buffer) {
numInvocations = 0;
prepareInvocationCounts(listener);
AppendOutputRunner runner = new AppendOutputRunner(listener);
for (String[] bufferElement: buffer) {
runner.appendBuffer(bufferElement[0], bufferElement[1], bufferElement[2]);
}
future = service.scheduleWithFixedDelay(runner, 0,
AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
long startTimeMs = System.currentTimeMillis();
while(numInvocations != numTimes) {
if (System.currentTimeMillis() - startTimeMs > 2000) {
fail("Buffered events were not sent for 2 seconds");
}
}
}
}

80
zeppelin-server/pom.xml
View file

@ -35,6 +35,7 @@
<properties>
<cxf.version>2.7.7</cxf.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
<hadoop-common.version>2.6.0</hadoop-common.version>
</properties>
<dependencyManagement>
@ -205,6 +206,61 @@
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop-common.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
@ -301,27 +357,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>src/test/java/com/webautomation/*</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>
@ -413,6 +448,9 @@
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencyManagement>
<dependencies>
<dependency>

100
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -38,6 +38,8 @@ import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
@ -317,7 +319,10 @@ public class NotebookRestApi {
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
String newNoteName = request.getName();
String newNoteName = null;
if (request != null) {
newNoteName = request.getName();
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
@ -477,7 +482,14 @@ public class NotebookRestApi {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
note.runAll();
try {
note.runAll();
} catch (Exception ex) {
LOG.error("Exception from run", ex);
return new JsonResponse<>(Status.PRECONDITION_FAILED,
ex.getMessage() + "- Not selected or Invalid Interpreter bind").build();
}
return new JsonResponse<>(Status.OK).build();
}
@ -558,7 +570,7 @@ public class NotebookRestApi {
}
/**
* Run paragraph job REST API
* Run asynchronously paragraph job REST API
*
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
@ -571,7 +583,7 @@ public class NotebookRestApi {
public Response runParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId, String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
LOG.info("run paragraph job asynchronously {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -584,22 +596,60 @@ public class NotebookRestApi {
}
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.setLastReplName(paragraph.getId());
note.persist(subject);
}
}
handleParagraphParams(message, note, paragraph);
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
/**
* Run synchronously a paragraph REST API
*
* @param noteId - noteId
* @param paragraphId - paragraphId
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("run/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraphSynchronously(@PathParam("notebookId") String noteId,
@PathParam("paragraphId") String paragraphId,
String message) throws
IOException, IllegalArgumentException {
LOG.info("run paragraph synchronously {} {} {}", noteId, paragraphId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
// handle params if presented
handleParagraphParams(message, note, paragraph);
if (paragraph.getListener() == null) {
note.initializeJobListenerForParagraph(paragraph);
}
paragraph.run();
final InterpreterResult result = paragraph.getResult();
if (result.code() == InterpreterResult.Code.SUCCESS) {
return new JsonResponse<>(Status.OK, result).build();
} else {
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, result).build();
}
}
/**
* Stop(delete) paragraph job REST API
*
@ -660,7 +710,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", request.getCronString());
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -688,7 +738,7 @@ public class NotebookRestApi {
Map<String, Object> config = note.getConfig();
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
return new JsonResponse<>(Status.OK).build();
}
@ -791,4 +841,20 @@ public class NotebookRestApi {
return new JsonResponse<>(Status.OK, notebooksFound).build();
}
private void handleParagraphParams(String message, Note note, Paragraph paragraph)
throws IOException {
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
}
}
}
}

37
zeppelin-server/src/main/java/org/apache/zeppelin/server/ActiveDirectoryGroupRealm.java
View file

@ -16,6 +16,10 @@
*/
package org.apache.zeppelin.server;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
@ -55,6 +59,13 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
private static final String ROLE_NAMES_DELIMETER = ",";
String KEYSTORE_PASS = "activeDirectoryRealm.systemPassword";
private String hadoopSecurityCredentialPath;
public void setHadoopSecurityCredentialPath(String hadoopSecurityCredentialPath) {
this.hadoopSecurityCredentialPath = hadoopSecurityCredentialPath;
}
/*--------------------------------------------
| I N S T A N C E V A R I A B L E S |
============================================*/
@ -91,13 +102,36 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
defaultFactory.setSearchBase(this.searchBase);
defaultFactory.setUrl(this.url);
defaultFactory.setSystemUsername(this.systemUsername);
defaultFactory.setSystemPassword(this.systemPassword);
defaultFactory.setSystemPassword(getSystemPassword());
this.ldapContextFactory = defaultFactory;
}
return this.ldapContextFactory;
}
private String getSystemPassword() {
String password = "";
if (StringUtils.isEmpty(this.hadoopSecurityCredentialPath)) {
password = this.systemPassword;
} else {
try {
Configuration configuration = new Configuration();
configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
this.hadoopSecurityCredentialPath);
CredentialProvider provider =
CredentialProviderFactory.getProviders(configuration).get(0);
CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(
KEYSTORE_PASS);
if (credEntry != null) {
password = new String(credEntry.getCredential());
}
} catch (Exception e) {
}
}
return password;
}
/**
* Builds an {@link AuthenticationInfo} object by querying the active directory LDAP context for
* the specified username. This method binds to the LDAP server using the provided username
@ -293,3 +327,4 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
}
}

68
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -336,7 +336,7 @@ public class NotebookServer extends WebSocketServlet implements
List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.id(), m);
broadcast(note.getId(), m);
}
}
}
@ -473,11 +473,11 @@ public class NotebookServer extends WebSocketServlet implements
for (Note note : notes) {
Map<String, String> info = new HashMap<>();
if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) {
if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNotebookId)) {
continue;
}
info.put("id", note.id());
info.put("id", note.getId());
info.put("name", note.getName());
notesInfo.add(info);
}
@ -486,7 +486,7 @@ public class NotebookServer extends WebSocketServlet implements
}
public void broadcastNote(Note note) {
broadcast(note.id(), new Message(OP.NOTE).put("note", note));
broadcast(note.getId(), new Message(OP.NOTE).put("note", note));
}
public void broadcastInterpreterBindings(String noteId,
@ -544,7 +544,7 @@ public class NotebookServer extends WebSocketServlet implements
notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -568,7 +568,7 @@ public class NotebookServer extends WebSocketServlet implements
userAndRoles, notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
@ -604,7 +604,7 @@ public class NotebookServer extends WebSocketServlet implements
note.setName(name);
note.setConfig(config);
if (cronUpdated) {
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
}
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
@ -643,7 +643,7 @@ public class NotebookServer extends WebSocketServlet implements
}
note.persist(subject);
addConnectionToNote(note.id(), (NotebookSocket) conn);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
broadcastNoteList(subject);
}
@ -697,7 +697,7 @@ public class NotebookServer extends WebSocketServlet implements
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
note.persist(subject);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
private void cloneNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -707,7 +707,7 @@ public class NotebookServer extends WebSocketServlet implements
String name = (String) fromMessage.get("name");
Note newNote = notebook.cloneNote(noteId, name, new AuthenticationInfo(fromMessage.principal));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.id(), (NotebookSocket) conn);
addConnectionToNote(newNote.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
broadcastNoteList(subject);
}
@ -810,12 +810,12 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(note.id()) == null) {
if (setting.getInterpreterGroup(note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(note.id()).getAngularObjectRegistry();
.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -852,17 +852,17 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(n.id()) == null) {
if (setting.getInterpreterGroup(n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(n.id()).getAngularObjectRegistry();
.getInterpreterGroup(n.getId()).getAngularObjectRegistry();
this.broadcastExcept(
n.id(),
n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", n.id())
.put("noteId", n.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -870,10 +870,10 @@ public class NotebookServer extends WebSocketServlet implements
}
} else { // broadcast to all web session for the note
this.broadcastExcept(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", ao.getParagraphId()),
conn);
}
@ -1130,10 +1130,8 @@ public class NotebookServer extends WebSocketServlet implements
.get("config");
p.setConfig(config);
// if it's the last paragraph, let's add a new one
boolean isTheLastParagraph = note.getLastParagraph().getId()
.equals(p.getId());
note.setLastReplName(paragraphId);
if (!(text.equals(note.getLastInterpreterName() + " ") || Strings.isNullOrEmpty(text)) &&
boolean isTheLastParagraph = note.isLastParagraph(p.getId());
if (!(text.trim().equals(p.getMagic()) || Strings.isNullOrEmpty(text)) &&
isTheLastParagraph) {
note.addParagraph();
}
@ -1149,7 +1147,7 @@ public class NotebookServer extends WebSocketServlet implements
new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()),
ex);
p.setStatus(Status.ERROR);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
}
}
@ -1309,7 +1307,7 @@ public class NotebookServer extends WebSocketServlet implements
@Override
public void onProgressUpdate(Job job, int progress) {
notebookServer.broadcast(
note.id(),
note.getId(),
new Message(OP.PROGRESS).put("id", job.getId()).put("progress",
job.progress()));
}
@ -1384,15 +1382,15 @@ public class NotebookServer extends WebSocketServlet implements
}
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id())
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId())
.getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.id());
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(note.id()).getId())
.put("noteId", note.id())
intpSetting.getInterpreterGroup(note.getId()).getId())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId())
));
}
@ -1413,7 +1411,7 @@ public class NotebookServer extends WebSocketServlet implements
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) {
if (object.getNoteId() != null && !note.getId().equals(object.getNoteId())) {
continue;
}
@ -1424,11 +1422,11 @@ public class NotebookServer extends WebSocketServlet implements
}
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId", interpreterGroupId)
.put("noteId", note.id())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId()));
}
}
@ -1438,7 +1436,7 @@ public class NotebookServer extends WebSocketServlet implements
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
if (noteId != null && !note.id().equals(noteId)) {
if (noteId != null && !note.getId().equals(noteId)) {
continue;
}
@ -1446,7 +1444,7 @@ public class NotebookServer extends WebSocketServlet implements
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(
note.id(),
note.getId(),
new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put(
"noteId", noteId).put("paragraphId", paragraphId));
}

11
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -95,28 +95,21 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
ZeppelinITUtils.sleep(1000, false);
waitForParagraph(1, "READY");
String oldIntpTag = driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created above",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.not(StringUtils.EMPTY));
CoreMatchers.equalTo(StringUtils.EMPTY));
setTextOfParagraph(1, " this is above ");
newPara = driver.findElement(By.xpath(getParagraphXPath(2) + "//div[contains(@class,'new-paragraph')][2]"));
action.moveToElement(newPara).click().build().perform();
waitForParagraph(3, "READY");
String lastIntpTag = driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created below",
driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.not(StringUtils.EMPTY));
CoreMatchers.equalTo(StringUtils.EMPTY));
setTextOfParagraph(3, " this is below ");
collector.checkThat("Compare interpreter name tag", oldIntpTag, CoreMatchers.equalTo(lastIntpTag));
collector.checkThat("The output field of paragraph1 contains",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.equalTo(" this is above "));

2
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -134,7 +134,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
Note note = ZeppelinServer.notebook.createNote(null);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
assertThat(get, isAllowed());
get.addRequestHeader("Origin", "http://localhost");
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {

27
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NotebookAuthorization;
@ -144,6 +145,32 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(note1.getId(), null);
}
@Test
public void testCloneNotebook() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
PostMethod post = httpPost("/notebook/" + note1.getId(), "");
LOG.info("testCloneNotebook response\n" + post.getResponseBodyAsString());
assertThat(post, isCreated());
Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
String clonedNotebookId = (String) resp.get("body");
post.releaseConnection();
GetMethod get = httpGet("/notebook/" + clonedNotebookId);
assertThat(get, isAllowed());
Map<String, Object> resp2 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
assertEquals((String)resp2Body.get("name"), "Note " + clonedNotebookId);
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(clonedNotebookId, null);
}
}

3
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
View file

@ -146,8 +146,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("compare note name", expectedNoteName, newNoteName);
assertEquals("initial paragraph check failed", 3, newNote.getParagraphs().size());
for (Paragraph p : newNote.getParagraphs()) {
if (StringUtils.isEmpty(p.getText()) ||
p.getText().trim().equals(newNote.getLastInterpreterName())) {
if (StringUtils.isEmpty(p.getText())) {
continue;
}
assertTrue("paragraph title check failed", p.getTitle().startsWith("title"));

14
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -79,7 +79,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55", p.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -91,7 +91,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
@ -118,7 +118,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[1] 3", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -141,7 +141,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -172,7 +172,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("10\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -204,7 +204,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p2.getStatus());
assertEquals("10", p2.getResult().message());
ZeppelinServer.notebook.removeNote(note.id(), null);
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -216,7 +216,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
if (isPyspark() && sparkVersionNumber >= 14) {
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
ZeppelinServer.notebook.getBindedInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {

127
zeppelin-web/CONTRIBUTING.md
View file

@ -1,127 +0,0 @@
# Contributing to Zeppelin-Web
## Dev Mode
When working on Zeppelin's WebApplication, it is recommended to run in dev mode.
For that, start Zeppelin server normally, then use ``./grunt serve`` in _zeppelin-web_ directory.
This will launch a Zeppelin WebApplication on port **9000** that will update on code changes.
## Technologies
Zeppelin WebApplication is using **AngularJS** as main Framework, and **Grunt** and **Bower** as helpers.
So you might want to get familiar with it.
[Here is a good start](http://www.sitepoint.com/kickstart-your-angularjs-development-with-yeoman-grunt-and-bower/)
(There is obviously plenty more ressources to learn)
## Coding style
* We follow mainly the [Google Javascript Guide](https://google-styleguide.googlecode.com/svn/trunk/javascriptguide.xml)
* We use a 2 spaces indentation
* We use single quotes
But don't worry, Eslint and Jscs will make you remember it for the most part.
We try not to have **JQuery except in directives**, If you want to include a library,
please search for its **angularJS** directive first.
If you still need to use it, then please use ``angular.element()`` instead of ``$()``
## Folder Structure & Code Organization
* `src` folder: Contains the Source code for Zeppelin WebApplication
* `dist` folder: Contains the compiled code after using **grunt build**
### Src and Code Organization
The `src` folder is organized as such:
<pre>
src/
├── app/
│ ├── name/
│ │ ├── name.controller.js
| | ├── name.html
| | ├── subComponent1/
| | | ├── subComponent1.html
| | | ├── subComponent1.css
│ | | └── subComponent1.controller.js
│ │ └── name.css
│ └── app.js
├── assets/
│ ├── images/
│ └── styles/
| ├── looknfeel/
│ └── printMode.css
├── components/
│ ├── component1/
| | ├── component1.html
│ | └── component1.controller.js
│ └── component2/
├── fonts/
| ├── *.{eot,svg,ttf,woff,otf}
│ └── *.css
├── favico.ico
├── index.html
└── 404.html
</pre>
The code is now organized in a component type of architecture, where everything is logically grouped.
#### File type name convention
In order to understand what is contained inside the .js files without opening it, we use some name conventions:
* .controller.js
* .directive.js
* .service.js
### Component Architecture
When we talk about Component architecture, we think about grouping files together in a logical way.
A component can then be made of multiple files like `.html`, `.css` or any other file type mentioned above.
Related components can be grouped as sub-component as long as they are used in that component only.
#### App folder
Contains the application `app.js` and page related components.
* Home Page
* Interpreter Page
* Notebook Page
etc...
The only resctiction being that a component in the `app` folder is **not used anywhere else**
#### Components folder
The `components` folder is here to contains any reusable component (used more than once)
### Fonts
Fonts files and their css are mixed together in the `fonts` folder
## New files includes
As we do not use yeoman to generate controllers or other type of files with this new structure,
we need to do some includes manually in `index.html` in order to use dev mode and compile correctly.
* Non-bower `.js` files needs to be injected between the tag `<!-- build:js({.tmp,src}) scripts/scripts.js -->`
* Css files needs to be injected between the tag `<!-- build:css(.tmp) styles/main.css -->`
## Add plugins with Bower
```
bower install <plugin> --save
```
The file index.html will automatically update with the new bower_component
<br/>
**Example**: `./bower install angular-nvd3`
You should find that line in the index.html file
```
<script src="bower_components/angular-nvd3/dist/angular-nvd3.js"></script>
````

2
zeppelin-web/README.md
View file

@ -61,4 +61,4 @@ also try to add proxy info to npm install command:
and retry to build again.
## Contribute on Zeppelin Web
If you wish to help us and contribute to Zeppelin WebApplication, please look at [Zeppelin WebApplication's contribution guideline](CONTRIBUTING.md).
If you wish to help us and contribute to Zeppelin WebApplication, please look at the overall project [contribution guidelines](https://zeppelin.apache.org/contribution/contributions.html) and the more focused [Zeppelin WebApplication's documentation](https://zeppelin.apache.org/contribution/webapplication.html).

9
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -325,6 +325,15 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
return;
}
if ($scope.newInterpreterSetting.name.indexOf('.') >= 0) {
BootstrapDialog.alert({
closable: true,
title: 'Add interpreter',
message: '\'.\' is invalid for interpreter name'
});
return;
}
if (_.findIndex($scope.interpreterSettings, {'name': $scope.newInterpreterSetting.name}) >= 0) {
BootstrapDialog.alert({
closable: true,

5
zeppelin-web/src/app/interpreter/interpreter.css
View file

@ -109,3 +109,8 @@
color: #000;
opacity: .5;
}
.remove-margin-top-bottom {
margin-top: 0px !important;
margin-bottom: 0px !important;
}

69
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -38,13 +38,17 @@ limitations under the License.
</div>
</div>
<div class="input-group col-md-4" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters"/>
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
<div class="row">
<div class="col-md-4">
<div class="input-group" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters"/>
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
</div>
</div>
</div>
</div>
@ -93,10 +97,9 @@ limitations under the License.
<small>
<span style="display:inline-block" ng-repeat="interpreter in setting.interpreterGroup"
title="{{interpreter.class}}">
<span ng-show="!$first">, </span>
%<span ng-show="!$parent.$first || $first">{{setting.name}}</span
><span ng-show="(!$parent.$first || $first) && !$first">.</span
><span ng-show="!$first">{{interpreter.name}}</span>
<span ng-show="!$first">, </span>%<span ng-show="!$parent.$first || $first">{{setting.name}}</span>
<span ng-show="(!$parent.$first || $first) && !$first">.</span>
<span ng-show="!$first">{{interpreter.name}}</span>
<span ng-show="$parent.$first && $first">(default)</span>
</span>
</small>
@ -170,37 +173,44 @@ limitations under the License.
</span>
<span>Interpreter for note</span>
</div>
<br />
</div>
<div class="row interpreter" style="margin-top: 5px;">
<div class="col-md-12">
<div class="checkbox">
<div class="checkbox remove-margin-top-bottom">
<span class="input-group" style="line-height:30px;">
<label><input type="checkbox" style="width:20px" id="isExistingProcess" ng-model="setting.option.isExistingProcess" ng-disabled="!valueform.$visible"/>
Connect to existing process </label>
<label>
<input type="checkbox" style="width:20px" id="isExistingProcess" ng-model="setting.option.isExistingProcess" ng-disabled="!valueform.$visible"/>
Connect to existing process
</label>
</span>
</div>
</div>
<div class="col-md-12" ng-show="setting.option.isExistingProcess">
</div>
<div class="row interpreter" ng-if="setting.option.isExistingProcess">
<div class="col-md-12">
<b>Host</b>
<input id="newInterpreterSettingHost" input pu-elastic-input
pu-elastic-input-minwidth="180px" ng-model="setting.option.host" ng-disabled="!valueform.$visible" />
</div>
<div class="col-md-12" ng-show="setting.option.isExistingProcess">
<div class="col-md-12">
<b>Port</b>
<input id="newInterpreterSettingPort" input pu-elastic-input
pu-elastic-input-minwidth="180px" ng-model="setting.option.port" ng-disabled="!valueform.$visible" />
</div>
</div>
<div class="row interpreter">
<div class="col-md-12">
<div class="checkbox">
<div class="checkbox remove-margin-top-bottom">
<span class="input-group" style="line-height:30px;">
<label><input type="checkbox" style="width:18px !important" id="idShowPermission" ng-click="togglePermissions(setting.name)" ng-model="setting.option.setPermission" ng-disabled="!valueform.$visible"/>
Set permission </label>
<label>
<input type="checkbox" style="width:20px !important" id="idShowPermission" ng-click="togglePermissions(setting.name)" ng-model="setting.option.setPermission" ng-disabled="!valueform.$visible"/>
Set permission
</label>
</span>
</div>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12">
<!-- permissions -->
<div ng-show="setting.option.setPermission" class="permissionsForm">
@ -223,14 +233,16 @@ limitations under the License.
<div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
<em>Currently there are no properties and dependencies set for this interpreter</em>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12" ng-show="!_.isEmpty(setting.properties) || valueform.$visible">
<h5>Properties</h5>
<table class="table table-striped">
<thead>
<tr>
<th style="width:30%">name</th>
<th>value</th>
<th ng-if="valueform.$visible">action</th>
<th style="width:40%">name</th>
<th style="width:40%">value</th>
<th style="width:20%" ng-if="valueform.$visible">action</th>
</tr>
</thead>
<tr ng-repeat="key in setting.properties | sortByKey" >
@ -263,7 +275,8 @@ limitations under the License.
</tr>
</table>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12" ng-show="!_.isEmpty(setting.dependencies) || valueform.$visible">
<h5>Dependencies</h5>
<p class="gray40-message" style="font-size:12px" ng-show="valueform.$visible">

21
zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html
View file

@ -20,37 +20,44 @@ limitations under the License.
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('table')}"
ng-click="setGraphMode('table', true)" ><i class="fa fa-table"></i>
ng-click="setGraphMode('table', true)"
tooltip="Table" tooltip-placement="bottom"><i class="fa fa-table"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('multiBarChart')}"
ng-click="setGraphMode('multiBarChart', true)"><i class="fa fa-bar-chart"></i>
ng-click="setGraphMode('multiBarChart', true)"
tooltip="Bar Chart" tooltip-placement="bottom"><i class="fa fa-bar-chart"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('pieChart')}"
ng-click="setGraphMode('pieChart', true)"><i class="fa fa-pie-chart"></i>
ng-click="setGraphMode('pieChart', true)"
tooltip="Pie Chart" tooltip-placement="bottom"><i class="fa fa-pie-chart"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('stackedAreaChart')}"
ng-click="setGraphMode('stackedAreaChart', true)"><i class="fa fa-area-chart"></i>
ng-click="setGraphMode('stackedAreaChart', true)"
tooltip="Area Chart" tooltip-placement="bottom"><i class="fa fa-area-chart"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('lineChart') || isGraphMode('lineWithFocusChart')}"
ng-click="paragraph.config.graph.lineWithFocus ? setGraphMode('lineWithFocusChart', true) : setGraphMode('lineChart', true)"><i class="fa fa-line-chart"></i>
ng-click="paragraph.config.graph.lineWithFocus ? setGraphMode('lineWithFocusChart', true) : setGraphMode('lineChart', true)"
tooltip="Line Chart" tooltip-placement="bottom"><i class="fa fa-line-chart"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('scatterChart')}"
ng-click="setGraphMode('scatterChart', true)"><i class="cf cf-scatter-chart"></i>
ng-click="setGraphMode('scatterChart', true)"
tooltip="Scatter Chart" tooltip-placement="bottom"><i class="cf cf-scatter-chart"></i>
</button>
<button type="button" class="btn btn-default btn-sm"
ng-if="paragraph.result.type == 'TABLE'"
ng-class="{'active': isGraphMode('map')}"
ng-click="setGraphMode('map', true)"><i class="fa fa-map-marker"></i>
ng-click="setGraphMode('map', true)"
tooltip="Map" tooltip-placement="bottom"><i class="fa fa-map-marker"></i>
</button>
<button type="button"

39
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -820,7 +820,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
var user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user;
var desc = 'Took ' + moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') +
'. Last updated by ' + user + ' at ' + moment(pdata.dateUpdated).format('MMMM DD YYYY, h:mm:ss A') + '.';
'. Last updated by ' + user + ' at ' + moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + '.';
if ($scope.isResultOutdated()) {
desc += ' (outdated)';
}
@ -865,6 +865,21 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
}
};
$scope.parseTableCell = function(cell) {
if (!isNaN(cell)) {
if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) {
return cell;
} else {
return Number(cell);
}
}
var d = moment(cell);
if (d.isValid()) {
return d;
}
return cell;
};
$scope.loadTableData = function(result) {
if (!result) {
return;
@ -898,8 +913,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
if (i === 0) {
columnNames.push({name: col, index: j, aggr: 'sum'});
} else {
cols.push(col);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: col});
var parsedCol = $scope.parseTableCell(col);
cols.push(parsedCol);
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: parsedCol});
}
}
if (i !== 0) {
@ -978,7 +994,9 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
cells: function(row, col, prop) {
var cellProperties = {};
cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) {
if (!isNaN(value)) {
if (value instanceof moment) {
td.innerHTML = value._i;
} else if (!isNaN(value)) {
cellProperties.format = '0,0.[00000]';
td.style.textAlign = 'left';
Handsontable.renderers.NumericRenderer.apply(this, arguments);
@ -1836,7 +1854,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
if (groups.length === 1 && values.length === 1) {
for (d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
colName = d3g[d3gIndex].key;
colName = colName.split('.')[0];
colName = colName.split('.').slice(0, -1).join('.');
d3g[d3gIndex].key = colName;
}
}
@ -2575,7 +2593,16 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
});
$scope.$on('appendParagraphOutput', function(event, data) {
if ($scope.paragraph.id === data.paragraphId) {
/* It has been observed that append events
* can be errorneously called even if paragraph
* execution has ended, and in that case, no append
* should be made. Also, it was observed that between PENDING
* and RUNNING states, append-events can be called and we can't
* miss those, else during the length of paragraph run, few
* initial output line/s will be missing.
*/
if ($scope.paragraph.id === data.paragraphId &&
($scope.paragraph.status === 'RUNNING' || $scope.paragraph.status === 'PENDING')) {
if ($scope.flushStreamingOutput) {
$scope.clearTextOutput();
$scope.flushStreamingOutput = false;

1
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -502,7 +502,6 @@ div.esri-view {
table.table-striped {
border-top: 1px solid #ddd;
margin-top: 20px;
}
.scroll-paragraph-down {

6
zeppelin-zengine/pom.xml
View file

@ -128,12 +128,6 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>

30
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -272,10 +272,15 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getKeyStorePath() {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH)));
String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
public String getKeyStoreType() {
@ -297,10 +302,13 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getTrustStorePath() {
String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
if (path == null) {
return getKeyStorePath();
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
return getRelativeDir(path);
return getRelativeDir(
String.format("%s/%s",
getConfDir(),
getString(path)));
}
}
@ -427,10 +435,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
public boolean getUseJdbcAlias() {
return getBoolean(ConfVars.ZEPPELIN_USE_JDBC_ALIAS);
}
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@ -557,9 +561,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true);
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000");
private String varName;
@SuppressWarnings("rawtypes")

3
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -506,6 +506,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
public InterpreterSetting createNewSetting(String name, String group,
List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException {
if (name.indexOf(".") >= 0) {
throw new IOException("'.' is invalid for InterpreterSetting name.");
}
InterpreterSetting setting = createFromInterpreterSettingRef(group);
setting.setName(name);
setting.setGroup(group);

131
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
@ -27,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import com.google.gson.Gson;
import org.apache.commons.lang.StringUtils;
@ -76,7 +77,6 @@ public class Note implements Serializable, ParagraphJobListener {
private String name = "";
private String id;
private AtomicReference<String> lastReplName = new AtomicReference<>(StringUtils.EMPTY);
private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create();
private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
@ -125,16 +125,6 @@ public class Note implements Serializable, ParagraphJobListener {
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
void putDefaultReplName() {
String defaultInterpreterName = getDefaultInterpreterName();
logger.info("defaultInterpreterName is '{}'", defaultInterpreterName);
lastReplName.set(defaultInterpreterName);
}
public String id() {
return id;
}
public String getId() {
return id;
}
@ -172,6 +162,28 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (paragraphNote.getId().equals(this.getId())) {
throw new IllegalArgumentException(format("The paragraph %s from note %s " +
"does not belong to note %s", paragraph.getId(), paragraphNote.getId(),
this.getId()));
}
boolean foundParagraph = false;
for (Paragraph ownParagraph : paragraphs) {
if (paragraph.getId().equals(ownParagraph.getId())) {
paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
foundParagraph = true;
}
}
if (!foundParagraph) {
throw new IllegalArgumentException(format("Cannot find paragraph %s " +
"from note %s", paragraph.getId(), paragraphNote.getId()));
}
}
void setJobListenerFactory(JobListenerFactory jobListenerFactory) {
this.jobListenerFactory = jobListenerFactory;
}
@ -202,7 +214,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph addParagraph() {
Paragraph p = new Paragraph(this, this, factory);
addLastReplNameIfEmptyText(p);
setParagraphMagic(p, paragraphs.size());
synchronized (paragraphs) {
paragraphs.add(p);
}
@ -258,7 +270,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph insertParagraph(int index) {
Paragraph p = new Paragraph(this, this, factory);
addLastReplNameIfEmptyText(p);
setParagraphMagic(p, index);
synchronized (paragraphs) {
paragraphs.add(index, p);
}
@ -268,22 +280,6 @@ public class Note implements Serializable, ParagraphJobListener {
return p;
}
/**
* Add Last Repl name If Paragraph has empty text
*
* @param p Paragraph
*/
private void addLastReplNameIfEmptyText(Paragraph p) {
String replName = lastReplName.get();
if (StringUtils.isEmpty(p.getText()) && StringUtils.isNotEmpty(replName)) {
p.setText(getInterpreterName(replName) + " ");
}
}
private String getInterpreterName(String replName) {
return StringUtils.isBlank(replName) ? StringUtils.EMPTY : "%" + replName;
}
/**
* Remove paragraph by id.
*
@ -292,7 +288,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph removeParagraph(String paragraphId) {
removeAllAngularObjectInParagraph(paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(id(), paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@ -354,8 +350,8 @@ public class Note implements Serializable, ParagraphJobListener {
if (index < 0 || index >= paragraphs.size()) {
if (throwWhenIndexIsOutOfBound) {
throw new IndexOutOfBoundsException("paragraph size is " + paragraphs.size() +
" , index is " + index);
throw new IndexOutOfBoundsException(
"paragraph size is " + paragraphs.size() + " , index is " + index);
} else {
return;
}
@ -411,7 +407,7 @@ public class Note implements Serializable, ParagraphJobListener {
List<Map<String, String>> paragraphsInfo = new LinkedList<>();
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
Map<String, String> info = populatePragraphInfo(p);
Map<String, String> info = populateParagraphInfo(p);
paragraphsInfo.add(info);
}
}
@ -422,14 +418,14 @@ public class Note implements Serializable, ParagraphJobListener {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return populatePragraphInfo(p);
return populateParagraphInfo(p);
}
}
return new HashMap<>();
}
}
private Map<String, String> populatePragraphInfo(Paragraph p) {
private Map<String, String> populateParagraphInfo(Paragraph p) {
Map<String, String> info = new HashMap<>();
info.put("id", p.getId());
info.put("status", p.getStatus().toString());
@ -445,15 +441,26 @@ public class Note implements Serializable, ParagraphJobListener {
return info;
}
private void setParagraphMagic(Paragraph p, int index) {
if (paragraphs.size() > 0) {
String magic;
if (index == 0) {
magic = paragraphs.get(0).getMagic();
} else {
magic = paragraphs.get(index - 1).getMagic();
}
if (StringUtils.isNotEmpty(magic)) {
p.setText(magic + "\n");
}
}
}
/**
* Run all paragraphs sequentially.
*/
public void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
synchronized (paragraphs) {
if (!paragraphs.isEmpty()) {
setLastReplName(paragraphs.get(paragraphs.size() - 1));
}
for (Paragraph p : paragraphs) {
if (!p.isEnabled()) {
continue;
@ -461,11 +468,7 @@ public class Note implements Serializable, ParagraphJobListener {
AuthenticationInfo authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUser(cronExecutingUser);
p.setAuthenticationInfo(authenticationInfo);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName());
intp.getScheduler().submit(p);
run(p.getId());
}
}
}
@ -480,16 +483,15 @@ public class Note implements Serializable, ParagraphJobListener {
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(getId(), requiredReplName);
if (intp == null) {
// TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter
if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) {
String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")");
logger.debug("New paragraph: {}", pText);
p.setEffectiveText(pText);
} else {
throw new InterpreterException("Interpreter " + requiredReplName + " not found");
}
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
intp.getScheduler().submit(p);
@ -584,16 +586,6 @@ public class Note implements Serializable, ParagraphJobListener {
repo.save(this, subject);
}
private void setLastReplName(Paragraph lastParagraphStarted) {
if (StringUtils.isNotEmpty(lastParagraphStarted.getRequiredReplName())) {
lastReplName.set(lastParagraphStarted.getRequiredReplName());
}
}
public void setLastReplName(String paragraphId) {
setLastReplName(getParagraph(paragraphId));
}
/**
* Persist this note with maximum delay.
*/
@ -602,7 +594,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
void unpersist(AuthenticationInfo subject) throws IOException {
repo.remove(id(), subject);
repo.remove(getId(), subject);
}
@ -658,14 +650,6 @@ public class Note implements Serializable, ParagraphJobListener {
this.info = info;
}
String getLastReplName() {
return lastReplName.get();
}
public String getLastInterpreterName() {
return getInterpreterName(getLastReplName());
}
@Override
public void beforeStatusChange(Job job, Status before, Status after) {
if (jobListenerFactory != null) {
@ -700,7 +684,6 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
@Override
public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) {
if (jobListenerFactory != null) {

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
View file

@ -36,7 +36,7 @@ public class NoteInfo {
}
public NoteInfo(Note note) {
id = note.id();
id = note.getId();
name = note.getName();
config = note.getConfig();
}

25
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -151,11 +151,10 @@ public class Notebook implements NoteEventListener {
Note note =
new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this);
synchronized (notes) {
notes.put(note.id(), note);
notes.put(note.getId(), note);
}
if (interpreterIds != null) {
bindInterpretersToNote(note.id(), interpreterIds);
note.putDefaultReplName();
bindInterpretersToNote(note.getId(), interpreterIds);
}
notebookIndex.addIndexDoc(note);
@ -239,10 +238,12 @@ public class Notebook implements NoteEventListener {
Note newNote = createNote(subject);
if (newNoteName != null) {
newNote.setName(newNoteName);
} else {
newNote.setName("Note " + newNote.getId());
}
// Copy the interpreter bindings
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.id());
bindInterpretersToNote(newNote.id(), boundInterpreterSettingsIds);
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);
List<Paragraph> paragraphs = sourceNote.getParagraphs();
for (Paragraph p : paragraphs) {
@ -419,15 +420,15 @@ public class Notebook implements NoteEventListener {
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.id(), note);
refreshCron(note.id());
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.id());
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
@ -508,11 +509,11 @@ public class Notebook implements NoteEventListener {
Collections.sort(noteList, new Comparator<Note>() {
@Override
public int compare(Note note1, Note note2) {
String name1 = note1.id();
String name1 = note1.getId();
if (note1.getName() != null) {
name1 = note1.getName();
}
String name2 = note2.id();
String name2 = note2.getId();
if (note2.getName() != null) {
name2 = note2.getName();
}
@ -593,14 +594,14 @@ public class Notebook implements NoteEventListener {
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.id());
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.id());
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )

63
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.notebook;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.helium.HeliumPackage;
@ -53,7 +54,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
private transient InterpreterFactory factory;
private transient Note note;
private transient AuthenticationInfo authenticationInfo;
private transient String effectiveText;
String title;
String text;
@ -114,14 +114,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
this.dateUpdated = new Date();
}
public void setEffectiveText(String effectiveText) {
this.effectiveText = effectiveText;
}
public String getEffectiveText() {
return effectiveText;
}
public AuthenticationInfo getAuthenticationInfo() {
return authenticationInfo;
}
@ -153,7 +145,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getRequiredReplName() {
return getRequiredReplName(null != effectiveText ? effectiveText : text);
return getRequiredReplName(text);
}
public static String getRequiredReplName(String text) {
@ -182,7 +174,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public String getScriptBody() {
return getScriptBody(null != effectiveText ? effectiveText : text);
return getScriptBody(text);
}
public static String getScriptBody(String text) {
@ -359,7 +351,6 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
} finally {
InterpreterContext.remove();
effectiveText = null;
}
}
@ -451,13 +442,13 @@ public class Paragraph extends Job implements Serializable, Cloneable {
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(note.id()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.id()).getResourcePool();
registry = intpGroup.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
@ -465,22 +456,22 @@ public class Paragraph extends Job implements Serializable, Cloneable {
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials = credentials.getUserCredentials(
authenticationInfo.getUser());
authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext = new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
note.getId(),
getId(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}
@ -575,4 +566,22 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
return scriptBody;
}
public String getMagic() {
String magic = StringUtils.EMPTY;
String text = getText();
if (text != null && text.startsWith("%")) {
magic = text.split("\\s+")[0];
if (isValidInterpreter(magic.substring(1))) {
return magic;
} else {
return StringUtils.EMPTY;
}
}
return magic;
}
private boolean isValidInterpreter(String replName) {
return factory.getInterpreter(note.getId(), replName) != null;
}
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -210,7 +210,7 @@ public class S3NotebookRepo implements NotebookRepo {
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json";
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -226,7 +226,7 @@ public class VFSNotebookRepo implements NotebookRepo {
FileObject rootDir = getRootDir();
FileObject noteDir = rootDir.resolveFile(note.id(), NameScope.CHILD);
FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD);
if (!noteDir.exists()) {
noteDir.createFolder();

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
View file

@ -176,7 +176,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
String notebook = GSON.toJson(note);
restApiClient.asyncPut(notebook);
LOG.info("ZeppelinHub REST API saving note {} ", note.id());
LOG.info("ZeppelinHub REST API saving note {} ", note.getId());
}
@Override

12
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -176,7 +176,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
factory.setInterpreters(note1.id(), factory.getDefaultInterpreterSettingList());
factory.setInterpreters(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -214,7 +214,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
@ -231,7 +231,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
}
// when unbind interpreter
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// then
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
@ -255,7 +255,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// Unbind all interpreter from note
// NullPointerException shouldn't occur here
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
// remove note
notebook.removeNote(note1.getId(), null);
@ -273,9 +273,9 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.id())) {
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
mock1IntpSettingId = setting.getId();
break;

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -157,4 +157,14 @@ public class InterpreterFactoryTest {
assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("note", "group1").getClassName());
}
@Test
public void testInvalidInterpreterSettingName() {
try {
factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
}
}
}

104
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
View file

@ -17,9 +17,6 @@
package org.apache.zeppelin.notebook;
import com.google.common.base.Optional;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSetting;
@ -83,92 +80,47 @@ public class NoteTest {
}
@Test
public void runJdbcTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("mysql"))).thenReturn(null);
when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter);
when(interpreter.getScheduler()).thenReturn(scheduler);
String pText = "%mysql show databases";
public void addParagraphWithEmptyReplNameTest() {
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p = note.addParagraph();
p.setText(pText);
note.run(p.getId());
ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class);
verify(scheduler, only()).submit(pCaptor.capture());
verify(interpreterFactory, times(2)).getInterpreter(anyString(), anyString());
assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText());
assertEquals("Change paragraph text", pText, pCaptor.getValue().getText());
}
@Test
public void putDefaultReplNameIfInterpreterSettingAbsent() {
when(interpreterFactory.getDefaultInterpreterSetting(anyString()))
.thenReturn(null);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName();
assertEquals(StringUtils.EMPTY, note.getLastReplName());
assertEquals(StringUtils.EMPTY, note.getLastInterpreterName());
}
@Test
public void putDefaultReplNameIfInterpreterSettingPresent() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(interpreterFactory.getDefaultInterpreterSetting(anyString()))
.thenReturn(interpreterSetting);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName();
assertEquals("spark", note.getLastReplName());
assertEquals("%spark", note.getLastInterpreterName());
}
@Test
public void addParagraphWithLastReplName() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(interpreterFactory.getDefaultInterpreterSetting(anyString()))
.thenReturn(interpreterSetting);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName(); //set lastReplName
Paragraph p = note.addParagraph();
assertEquals("%spark ", p.getText());
assertNull(p.getText());
}
@Test
public void insertParagraphWithLastReplName() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(interpreterFactory.getDefaultInterpreterSetting(anyString()))
.thenReturn(interpreterSetting);
public void addParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
note.putDefaultReplName(); //set lastReplName
Paragraph p1 = note.addParagraph();
p1.setText("%spark ");
Paragraph p2 = note.addParagraph();
Paragraph p = note.insertParagraph(note.getParagraphs().size());
assertEquals("%spark ", p.getText());
assertEquals("%spark\n", p2.getText());
}
@Test
public void setLastReplName() {
String paragraphId = "HelloWorld";
Note note = Mockito.spy(new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener));
Paragraph mockParagraph = Mockito.mock(Paragraph.class);
when(note.getParagraph(paragraphId)).thenReturn(mockParagraph);
when(mockParagraph.getRequiredReplName()).thenReturn("spark");
public void insertParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter);
note.setLastReplName(paragraphId);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph();
p1.setText("%spark ");
Paragraph p2 = note.insertParagraph(note.getParagraphs().size());
assertEquals("spark", note.getLastReplName());
assertEquals("%spark\n", p2.getText());
}
@Test
public void insertParagraphWithInvalidReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("invalid"))).thenReturn(null);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph();
p1.setText("%invalid ");
Paragraph p2 = note.insertParagraph(note.getParagraphs().size());
assertNull(p2.getText());
}
}

82
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -171,7 +171,7 @@ public class NotebookTest implements JobListenerFactory{
notebook.reloadAllNotes(null);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 2);
assertEquals(notes.get(1).id(), copiedNote.id());
assertEquals(notes.get(1).getId(), copiedNote.getId());
assertEquals(notes.get(1).getName(), copiedNote.getName());
assertEquals(notes.get(1).getParagraphs(), copiedNote.getParagraphs());
@ -283,13 +283,13 @@ public class NotebookTest implements JobListenerFactory{
config.put("enabled", true);
config.put("cron", "* * * * * ?");
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
Thread.sleep(1*1000);
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
Thread.sleep(1000);
dateFinished = p.getDateFinished();
assertNotNull(dateFinished);
@ -318,7 +318,7 @@ public class NotebookTest implements JobListenerFactory{
config.put("cron", "1/3 * * * * ?");
config.put("releaseresource", true);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
@ -342,7 +342,7 @@ public class NotebookTest implements JobListenerFactory{
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
notebook.refreshCron(note.getId());
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
@ -398,6 +398,16 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(cp.getResult().message(), p.getResult().message());
}
@Test
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note cloneNote = notebook.cloneNote(note.getId(), null, null);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
}
@Test
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
@ -445,7 +455,7 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(1, ResourcePoolUtils.getAllResources().size());
// remove note
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
}
@ -463,20 +473,20 @@ public class NotebookTest implements JobListenerFactory{
Paragraph p1 = note.addParagraph();
// add paragraph scope object
registry.add("o1", "object1", note.id(), p1.getId());
registry.add("o1", "object1", note.getId(), p1.getId());
// add notebook scope object
registry.add("o2", "object2", note.id(), null);
registry.add("o2", "object2", note.getId(), null);
// add global scope object
registry.add("o3", "object3", null, null);
// remove notebook
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
// notebook scope or paragraph scope object should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o2", note.id(), p1.getId()));
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", note.getId(), p1.getId()));
// global object sould be remained
assertNotNull(registry.get("o3", null, null));
@ -496,10 +506,10 @@ public class NotebookTest implements JobListenerFactory{
Paragraph p1 = note.addParagraph();
// add paragraph scope object
registry.add("o1", "object1", note.id(), p1.getId());
registry.add("o1", "object1", note.getId(), p1.getId());
// add notebook scope object
registry.add("o2", "object2", note.id(), null);
registry.add("o2", "object2", note.getId(), null);
// add global scope object
registry.add("o3", "object3", null, null);
@ -508,10 +518,10 @@ public class NotebookTest implements JobListenerFactory{
note.removeParagraph(p1.getId());
// paragraph scope should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o1", note.getId(), null));
// notebook scope and global object sould be remained
assertNotNull(registry.get("o2", note.id(), null));
assertNotNull(registry.get("o2", note.getId(), null));
assertNotNull(registry.get("o3", null, null));
}
@ -527,7 +537,7 @@ public class NotebookTest implements JobListenerFactory{
.getAngularObjectRegistry();
// add local scope object
registry.add("o1", "object1", note.id(), null);
registry.add("o1", "object1", note.getId(), null);
// add global scope object
registry.add("o2", "object2", null, null);
@ -537,9 +547,9 @@ public class NotebookTest implements JobListenerFactory{
.getAngularObjectRegistry();
// local and global scope object should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
}
@Test
@ -548,43 +558,43 @@ public class NotebookTest implements JobListenerFactory{
Note note = notebook.createNote(null);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
// empty owners, readers and writers means note is public
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
notebookAuthorization.setOwners(note.id(),
notebookAuthorization.setOwners(note.getId(),
new HashSet<String>(Arrays.asList("user1")));
notebookAuthorization.setReaders(note.id(),
notebookAuthorization.setReaders(note.getId(),
new HashSet<String>(Arrays.asList("user1", "user2")));
notebookAuthorization.setWriters(note.id(),
notebookAuthorization.setWriters(note.getId(),
new HashSet<String>(Arrays.asList("user1")));
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), false);
assertEquals(notebookAuthorization.isOwner(note.id(),
assertEquals(notebookAuthorization.isOwner(note.getId(),
new HashSet<String>(Arrays.asList("user1"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user3"))), false);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), false);
assertEquals(notebookAuthorization.isWriter(note.id(),
assertEquals(notebookAuthorization.isWriter(note.getId(),
new HashSet<String>(Arrays.asList("user1"))), true);
// Test clearing of permssions
notebookAuthorization.setReaders(note.id(), Sets.<String>newHashSet());
assertEquals(notebookAuthorization.isReader(note.id(),
notebookAuthorization.setReaders(note.getId(), Sets.<String>newHashSet());
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user2"))), true);
assertEquals(notebookAuthorization.isReader(note.id(),
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user3"))), true);
notebook.removeNote(note.id(), null);
notebook.removeNote(note.getId(), null);
}
@Test
@ -777,8 +787,8 @@ public class NotebookTest implements JobListenerFactory{
note1.removeParagraph(p1.getId());
assertEquals(1, onParagraphRemove.get());
List<String> settings = notebook.getBindedInterpreterSettingsIds(note1.id());
notebook.bindInterpretersToNote(note1.id(), new LinkedList<String>());
List<String> settings = notebook.getBindedInterpreterSettingsIds(note1.getId());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
assertEquals(settings.size(), unbindInterpreter.get());
notebook.removeNote(note1.getId(), null);

26
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -73,32 +73,6 @@ public class ParagraphTest {
assertEquals("md", Paragraph.getRequiredReplName(text));
}
@Test
public void effectiveTextTest() {
InterpreterFactory interpreterFactory = mock(InterpreterFactory.class);
Interpreter interpreter = mock(Interpreter.class);
Note note = mock(Note.class);
Paragraph p = new Paragraph("paragraph", note, null, interpreterFactory);
p.setText("%h2 show databases");
p.setEffectiveText("%jdbc(h2) show databases");
assertEquals("Get right replName", "jdbc", p.getRequiredReplName());
assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody());
when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter);
when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE);
when(note.getId()).thenReturn("noteId");
try {
p.jobRun();
} catch (Throwable throwable) {
// Do nothing
}
assertEquals("Erase effective Text", "h2", p.getRequiredReplName());
assertEquals("Erase effective Text", "show databases", p.getScriptBody());
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given

6
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
View file

@ -224,7 +224,7 @@ public class GitNotebookRepoTest {
Map<String, Object> config = p1.getConfig();
config.put("enabled", true);
p1.setConfig(config);
p1.setText("%md checkpoint test text");
p1.setText("checkpoint test text");
notebookRepo.save(note, null);
// second checkpoint
@ -245,7 +245,7 @@ public class GitNotebookRepoTest {
Paragraph p2 = note.addParagraph();
config.put("enabled", false);
p2.setConfig(config);
p2.setText("%md get revision when modified note test text");
p2.setText("get revision when modified note test text");
notebookRepo.save(note, null);
note = notebookRepo.get(TEST_NOTE_ID, null);
int paragraphCount_3 = note.getParagraphs().size();
@ -282,7 +282,7 @@ public class GitNotebookRepoTest {
Map<String, Object> config = p1.getConfig();
config.put("enabled", true);
p1.setConfig(config);
p1.setText("%md get revision when modified note test text");
p1.setText("get revision when modified note test text");
notebookRepo.save(note, null);
int paragraphCount_2 = note.getParagraphs().size();

Some files were not shown because too many files have changed in this diff Show more