Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into windows-support

Browse source 
resolved conflict in docs/install/install.md
This commit is contained in:
Silvio Fiorito 2016-03-03 01:54:42 -05:00
commit 82acdcf4eb
166 changed files with 6425 additions and 1806 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.github/PULL_REQUEST_TEMPLATE vendored Normal file
View file

@ -0,0 +1,24 @@
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md
### What type of PR is it?
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update?
* Is there breaking changes for older versions?
* Does this needs documentation?

8
.gitignore vendored
View file

@ -8,6 +8,13 @@
# interpreter
/interpreter/
# interpreter temp files
spark/derby.log
spark/metastore_db
spark-1.*-bin-hadoop*
lens/lens-cli-hist.log
# conf file
conf/zeppelin-env.sh
conf/zeppelin-env.cmd
@ -46,6 +53,7 @@ zeppelin-web/bower_components
**/data/
**/build/
**/testing/
!/testing/
# OS generated files #
######################

21
.travis.yml
View file

@ -19,27 +19,31 @@ matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.1
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.0
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "export DISPLAY=:99.0"
@ -49,11 +53,12 @@ install:
- mvn $BUILD_FLAG $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
after_failure:
- cat target/rat.txt

21
CONTRIBUTING.md
View file

@ -7,12 +7,17 @@ Contributing to Zeppelin (Source code, Documents, Image, Website) means you agre
2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
3. Contribute your patch via Pull Request.
Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it.
For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful.
## Creating a Pull Request
In order to make the review process easier, please follow this template when making a Pull Request:
```
### What is this PR for?
A few sentences describing the overall goals of the pull request's commits.
First time? Check out the contributing guide - https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md
### What type of PR is it?
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
@ -20,7 +25,9 @@ A few sentences describing the overall goals of the pull request's commits.
### Todos
* [ ] - Task
### Is there a relevant Jira issue?
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
@ -33,12 +40,6 @@ Outline the steps to test the PR here.
* Does this needs documentation?
```
You can also use this small bookmarklet tool to fill your Pull Request fields automatically:
```
javascript:(function() {var e = document.getElementById('pull_request_body');if (e) {e.value += '### What is this PR for?\nA few sentences describing the overall goals of the pull request\'s commits.\n\n### What type of PR is it?\n[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]\n\n### Todos\n* [ ] - Task\n\n### Is there a relevant Jira issue?\n\n### How should this be tested?\nOutline the steps to test the PR here.\n\n### Screenshots (if appropriate)\n\n### Questions:\n* Does the licenses files need update?\n* Is there breaking changes for older versions?\n* Does this needs documentation?';}})();
```
## Testing a Pull Request
You can also test and review a particular Pull Request. Here are two useful ways.
@ -144,13 +145,13 @@ First of all, you need the Zeppelin source code. The official location for Zeppe
Get the source code on your development machine using git.
```
git clone http://git.apache.org/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 http://git.apache.org/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
or with write access

2
NOTICE
View file

@ -1,5 +1,5 @@
Apache Zeppelin (incubating)
Copyright 2015 The Apache Software Foundation
Copyright 2015 - 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

2
README.md
View file

@ -104,7 +104,7 @@ minor version can be adjusted by `-Dhadoop.version=x.x.x`
##### -Pyarn (optional)
enable YARN support for local mode
> YARN for local mode is not supported for Spark v1.5.0 or higher. Set SPARK_HOME instead.
##### -Ppyspark (optional)

9
SECURITY-README.md
View file

@ -33,11 +33,8 @@ The scope of this PR is to require credentials to access Zeppelin. To achieve th
Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info).
## Websocket security
Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach:
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above).

4
bin/common.sh
View file

@ -36,10 +36,6 @@ if [[ -z "${ZEPPELIN_LOG_DIR}" ]]; then
export ZEPPELIN_LOG_DIR="${ZEPPELIN_HOME}/logs"
fi
if [[ -z "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
export ZEPPELIN_NOTEBOOK_DIR="${ZEPPELIN_HOME}/notebook"
fi
if [[ -z "$ZEPPELIN_PID_DIR" ]]; then
export ZEPPELIN_PID_DIR="${ZEPPELIN_HOME}/run"
fi

2
bin/functions.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information

22
bin/zeppelin-daemon.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@ -21,7 +19,7 @@
# description: Start and stop daemon script for.
#
USAGE="Usage: zeppelin-daemon.sh [--config <conf-dir>] {start|stop|restart|reload|status}"
USAGE="Usage: zeppelin-daemon.sh [--config <conf-dir>] {start|stop|upstart|restart|reload|status}"
if [[ "$1" == "--config" ]]; then
shift
@ -93,11 +91,6 @@ function initialize_default_directories() {
echo "Pid dir doesn't exist, create ${ZEPPELIN_PID_DIR}"
$(mkdir -p "${ZEPPELIN_PID_DIR}")
fi
if [[ ! -d "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
echo "Notebook dir doesn't exist, create ${ZEPPELIN_NOTEBOOK_DIR}"
$(mkdir -p "${ZEPPELIN_NOTEBOOK_DIR}")
fi
}
function wait_for_zeppelin_to_die() {
@ -159,6 +152,16 @@ function check_if_process_is_alive() {
fi
}
function upstart() {
# upstart() allows zeppelin to be run and managed as a service
# for example, this could be called from an upstart script in /etc/init
# where the service manager starts and stops the process
initialize_default_directories
$ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_MAIN >> "${ZEPPELIN_OUTFILE}"
}
function start() {
local pid
@ -241,6 +244,9 @@ case "${1}" in
stop)
stop
;;
upstart)
upstart
;;
reload)
stop
start

2
bin/zeppelin.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information

5
conf/shiro.ini
View file

@ -19,8 +19,9 @@
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password1
user1 = password2
user2 = password3
user1 = password2, role1, role2
user2 = password3, role3
user3 = password4, role2
# Sample LDAP configuration, for user Authentication, currently tested for single Realm
[main]

27
conf/zeppelin-site.xml.template
View file

@ -83,6 +83,33 @@
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>
<name>zeppelin.notebook.azure.user</name>
<value>user</value>
<description>optional user name for Azure folder structure</description>
</property>
<property>
<name>zeppelin.notebook.azure.share</name>
<value>zeppelin</value>
<description>share name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.azure.connectionString</name>
<value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
<description>share name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.AzureNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
-->
<!-- For versioning your local norebook storage using Git repository
<property>
<name>zeppelin.notebook.storage</name>

39
docs/README.md
View file

@ -8,10 +8,13 @@ See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
**tl;dr version:**
```
ruby --version >= 1.9.3
gem install bundler
# go to /docs under your Zeppelin source
bundle install
```
*On OS X 10.9 you may need to do "xcode-select --install"*
@ -20,28 +23,28 @@ See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
bundle exec jekyll serve --watch
## Deploy to ASF svnpubsub infra (commiters only)
1. generate static website in `./_site`
```
bundle exec jekyll build --safe
```
2. checkout ASF repo
```
svn co https://svn.apache.org/repos/asf/incubator/zeppelin asf-zepplelin
```
3. copy zeppelin/_site to asf-zepplelin/site/docs/[VERSION]
4. ```svn commit```
## Adding a new page
rake page name="new-page.md"
## Bumping up version in a new release
## Bumping up version
* `ZEPPELIN_VERSION` and `BASE_PATH` property in _config.yml
* `Zeppelin <small>([VERSION])</small>` in _includes/themes/zeppelin/_navigation.html
should be updated
* `BASE_PATH` property in _config.yml
* `ZEPPELIN <small>([VERSION])</small>` in _includes/themes/zeppelin/_navigation.html
need to be updated
## Deploy to ASF svnpubsub infra (for committers only)
1. generate static website in `./_site`
```
# go to /docs under Zeppelin source
bundle exec jekyll build --safe
```
2. checkout ASF repo
```
svn co https://svn.apache.org/repos/asf/incubator/zeppelin asf-zeppelin
```
3. copy `zeppelin/docs/_site` to `asf-zeppelin/site/docs/[VERSION]`
4. ```svn commit```

2
docs/_config.yml
View file

@ -21,6 +21,8 @@ author :
twitter : ASF
feedburner : feedname
ZEPPELIN_VERSION : 0.6.0-incubating-SNAPSHOT
# The production_url is only used when full-domain names are needed
# such as sitemap.txt
# Most places will/should use BASE_PATH to make the urls

15
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -26,11 +26,14 @@
<li><a href="{{BASE_PATH}}/install/yarn_install.html">YARN Install</a></li>
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Virtual Machine Install</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/install/upgrade.html">Upgrade Version</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Tutorial</b><span></li -->
<li><a href="{{BASE_PATH}}/tutorial/tutorial.html">Tutorial</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Guide</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/dynamicform.html">Dynamic Form</a></li>
<li><a href="{{BASE_PATH}}/manual/publish.html">Publish your Paragraph</a></li>
</ul>
</li>
<li>
@ -45,6 +48,7 @@
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
@ -82,13 +86,22 @@
<li><a href="{{BASE_PATH}}/rest-api/rest-notebook.html">Notebook API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-configuration.html">Configuration API</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Security</b><span></li -->
<li><a href="{{BASE_PATH}}/security/overview.html">Security Overview</a></li>
<li><a href="{{BASE_PATH}}/security/authentication.html">Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/interpreter_authorization.html">Interpreter Authorization</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Development</b><span></li -->
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Shiro Security</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/shiroauthentication.html">Shiro Authentication</a></li>
</ul>
</li>
</ul>
</nav><!--/.navbar-collapse -->
</div>
</div>
</div>

BIN
docs/assets/themes/zeppelin/img/docs-img/copy-the-link.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 185 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/jdbc-multi-connection-setting.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 104 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/jdbc-simple-connection-setting.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 80 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 7 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/link-the-paragraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 311 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/your-website.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 425 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin-login.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/form_checkbox.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 22 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/form_checkbox_delimiter.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 18 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/form_checkbox_prog.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 37 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/interpreter_binding.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 227 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/interpreter_persession.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 37 KiB

58
docs/development/howtocontribute.md
View file

@ -5,52 +5,49 @@ description: "How to contribute"
group: development
---
# Contributing to Apache Zeppelin ( Code )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
## Setting up
Here are some tools you will need to build and test Zeppelin.
#### Software Configuration Management ( SCM )
### Setting up
Here are some things you will need to do to build and test Zeppelin.
Since Zeppelin uses Git for it's SCM system, you need git client installed in your development machine.
#### Software Configuration Management(SCM)
Zeppelin uses Git for it's SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` You'll need git client installed in your development machine.
#### Integrated Development Environment(IDE)
#### Integrated Development Environment ( IDE )
You are free to use whatever IDE you prefer, or your favorite command line editor.
#### Build Tools
### Build Tools
To build the code, install
Oracle Java 7
Apache Maven
* Oracle Java 7
* Apache Maven
### Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
## Getting the source code
First of all, you need Zeppelin source code. The official location of Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
#### git access
### git access
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 https://github.com/apache/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
#### Fork repository
If you want not only build Zeppelin but also make changes, then you need to fork Zeppelin repository and make pull request.
If you want not only build Zeppelin but also make any changes, then you need fork [Zeppelin github mirror repository](https://github.com/apache/incubator-zeppelin) and make a pull request.
###Build
@ -67,7 +64,7 @@ mvn install -DskipTests
To build with specific spark / hadoop version
```
mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x
```
### Run Zeppelin server in development mode
@ -76,7 +73,8 @@ mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
NOTE: make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory otherwise your server build will fail to find the required dependencies in the local repro
> **Note:** Make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro.
or use daemon script
@ -84,15 +82,13 @@ or use daemon script
bin/zeppelin-daemon start
```
Server will be run on http://localhost:8080
Server will be run on [http://localhost:8080](http://localhost:8080).
### Generating Thrift Code
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this, but if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
To regenerate the code, install thrift-0.9.0 and change directory into Zeppelin source directory. and then run following command
To regenerate the code, install **thrift-0.9.0** and change directory into Zeppelin source directory. and then run following command
```
@ -100,10 +96,10 @@ thrift -out zeppelin-interpreter/src/main/java/ --gen java zeppelin-interpreter/
```
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

37
docs/development/howtocontributewebsite.md
View file

@ -5,62 +5,51 @@ description: "How to contribute (website)"
group: development
---
## IMPORTANT
# Contributing to Apache Zeppelin ( Website )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
## Modifying the website
### Modifying the website
<br />
#### Getting the source code
Website is hosted in 'master' branch under `/docs/` dir.
First of all, you need the website source code. The official location of mirror for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin).
First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git
git clone git://git.apache.org/incubator-zeppelin.git
cd docs
```
<br />
#### Build
To build, you'll need to install some prerequisites.
To build, you'll need to install some prerequisites. Please check 'Build documentation' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build-documentation).
Please check 'Build' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build)
<br />
#### Run website in development mode
While you're modifying website, you'll want to see preview of it.
While you're modifying website, you'll want to see preview of it. Please check 'Run website' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run-website).
Please check 'Run' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run)
You'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser.
You'll be able to access it on localhost:4000 with your webbrowser.
#### Making a Pull Request
<br />
#### Pull request
When you're ready, just make a pull-request.
When you are ready, just make a pull-request.
<br />
### Alternative way
## Alternative way
You can directly edit .md files in `/docs/` dir at github's web interface and make pull-request immediatly.
<br />
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

8
docs/development/writingzeppelininterpreter.md
View file

@ -22,12 +22,16 @@ limitations under the License.
### What is Zeppelin Interpreter
Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need scala interpreter.
Every Interpreter belongs to an InterpreterGroup. InterpreterGroup is a unit of start/stop interpreter.
Every Interpreter belongs to an InterpreterGroup.
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/interpreter.png" />
All Interpreters in the same interpreter group are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via thrift.
InterpreterSetting is configuration of a given InterpreterGroup and a unit of start/stop interpreter.
All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via thrift.
In 'Separate Interpreter for each note' mode, new Interpreter instance will be created per notebook. But it still runs on the same JVM while they're in the same InterpreterSettings.
### Make your own Interpreter

8
docs/index.md
View file

@ -37,7 +37,6 @@ limitations under the License.
<div class="col-md-7"><img class="img-responsive" style="border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/notebook.png" /></div>
</div>
<br />
### Multiple language backend
@ -86,7 +85,6 @@ With simple drag and drop Zeppelin aggeregates the values and display them in pi
</div>
Learn more about Zeppelin's Display system. ( [text](./displaysystem/display.html), [html](./displaysystem/display.html#html), [table](./displaysystem/table.html), [angular](./displaysystem/angular.html) )
<br />
### Dynamic forms
@ -96,7 +94,6 @@ Zeppelin can dynamically create some input forms into your notebook.
Learn more about [Dynamic Forms](./manual/dynamicform.html).
<br />
### Collaboration
@ -114,6 +111,8 @@ This way, you can easily embed it as an iframe inside of your website.</p>
<img class="img-responsive center-block" src="/assets/themes/zeppelin/img/screenshots/publish.png" />
</div>
If you want to learn more about this feature, please visit [this page](./manual/publish.html).
<br />
### 100% Opensource
@ -124,5 +123,4 @@ Join the [Mailing list](./community.html) and report issues on our [Issue tracke
<br />
### Undergoing Incubation
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.

73
docs/install/install.md
View file

@ -22,9 +22,9 @@ limitations under the License.
## Zeppelin Installation
Welcome to your first trial to explore Zeppelin !
Welcome to your first trial to explore Zeppelin!
In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the **Zeppelin Configuration** section below.
In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below.
### Install with Binary Package
@ -32,9 +32,17 @@ If you want to install Zeppelin with latest binary package, please visit [this p
### Build from Zeppelin Source
You can also build Zeppelin from the source. Please check instructions in `README.md` in [Zeppelin github](https://github.com/apache/incubator-zeppelin/blob/master/README.md).
You can also build Zeppelin from the source.
#### Prerequisites for build
* Java 1.7
* Git
* Maven(3.1.x or higher)
* Node.js Package Manager
If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/incubator-zeppelin/blob/master/README.md) for the details.
<a name="zeppelin-configuration"> </a>
## Zeppelin Configuration
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
@ -75,6 +83,12 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>zeppelin.server.allowed.origins</td>
<td>*</td>
<td>Enables a way to specify a ',' separated list of allowed origins for rest and websockets. <br /> i.e. http://localhost:8080 </td>
</tr>
<tr>
<td>N/A</td>
<td>zeppelin.anonymous.allowed</td>
<td>true</td>
<td>Anonymous user is allowed by default.</td>
</tr>
<tr>
<td>ZEPPELIN_SERVER_CONTEXT_PATH</td>
@ -172,6 +186,24 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>user</td>
<td>A user name of S3 bucket<br />i.e. <code>bucket/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=&lt;accountName&gt;;AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Share where the Zeppelin notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
<td>zeppelin.notebook.azure.user</td>
<td>user</td>
<td>An optional user name of Azure file share<br />i.e. <code>share/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_STORAGE</td>
<td>zeppelin.notebook.storage</td>
@ -212,8 +244,43 @@ After successful start, visit [http://localhost:8080](http://localhost:8080) wit
bin/zeppelin-daemon.sh stop
```
#### Start Zeppelin with a service manager such as upstart
Zeppelin can auto start as a service with an init script, such as services managed by upstart.
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
This example has been tested with Ubuntu Linux.
This also allows the service to be managed with commands such as
`sudo service zeppelin start`
`sudo service zeppelin stop`
`sudo service zeppelin restart`
Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart`
##### zeppelin.conf
```
description "zeppelin"
start on (local-filesystems and net-device-up IFACE!=lo)
stop on shutdown
# Respawn the process on unexpected termination
respawn
# respawn the job up to 7 times within a 5 second period.
# If the job exceeds these values, it will be stopped and marked as failed.
respawn limit 7 5
# zeppelin was installed in /usr/share/zeppelin in this example
chdir /usr/share/zeppelin
exec bin/zeppelin-daemon.sh upstart
```
#### Running on Windows
```
bin\zeppelin.cmd
```

44
docs/install/upgrade.md Normal file
View file

@ -0,0 +1,44 @@
---
layout: page
title: "Manual upgrade procedure for Zeppelin"
description: ""
group: install
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## Manual upgrade procedure for Zeppelin
Basically, newer version of Zeppelin works with previous version notebook directory and configurations.
So, copying `notebook` and `conf` directory should be enough.
### Instructions
1. Stop Zeppelin
```
bin/zeppelin-daemon.sh stop
```
1. Copy your `notebook` and `conf` directory into a backup directory
1. Download newer version of Zeppelin and Install. See [Install page](./install.html)
1. Copy backup `notebook` and `conf` directory into newer version of Zeppelin `notebook` and `conf` directory
1. Start Zeppelin
```
bin/zeppelin-daemon.sh start
```

46
docs/install/virtual_machine.md
View file

@ -22,11 +22,11 @@ limitations under the License.
## Vagrant Virtual Machine for Apache Zeppelin
The Apache Zeppelin distribution includes a scripts directory
Apache Zeppelin distribution includes a scripts directory
`scripts/vagrant/zeppelin-dev`
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras)
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras).
####Installing the required components to launch a virtual machine.
@ -34,19 +34,22 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
If you are running Windows and don't yet have python installed, [install Python 2.7.x](https://www.python.org/downloads/release/python-2710/) first.
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
```
sudo easy_install pip
sudo pip install ansible
ansible --version
```
After then, please check whether it reports **ansible version 1.9.2 or higher**.
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
Thats it!
You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
Thats it ! You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
If you don't wish to build Zeppelin from scratch, run the z-manager installer script while running in the guest VM:
@ -55,18 +58,17 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
```
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning Zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the Zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -74,7 +76,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -117,15 +119,15 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
### [Python Extras](id:pythonextras)
With zeppelin running, Numpy, SciPy, Pandas and Matplotlib will be available. Create a pyspark notebook, and try
With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available. Create a pyspark notebook, and try the below code.
```
```python
%pyspark
import numpy
@ -139,9 +141,9 @@ print "pandas " + pandas.__version__
print "matplotlib " + matplotlib.__version__
```
To Test plotting using matplotlib into a rendered %html SVG image, try
To Test plotting using Matplotlib into a rendered `%html` SVG image, try
```
```python
%pyspark
import matplotlib

132
docs/install/yarn_install.md
View file

@ -20,7 +20,7 @@ limitations under the License.
{% include JB/setup %}
## Introduction
This page describes how to pre-configure a bare metal node, build & configure Zeppelin on it, configure Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop. It also describes steps to configure Spark & Hive interpreter of Zeppelin.
This page describes how to pre-configure a bare metal node, configure Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop. It also describes steps to configure Spark & Hive interpreter of Zeppelin.
## Prepare Node
@ -44,84 +44,16 @@ Its assumed in the rest of the document that zeppelin user is indeed created and
### List of Prerequisites
* CentOS 6.x
* Git
* Java 1.7
* Apache Maven
* Hadoop client.
* Spark.
* CentOS 6.x, Mac OSX, Ubuntu 14.X
* Java 1.7
* Hadoop client
* Spark
* Internet connection is required.
Its assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine. The working directory of all prerequisite pacakges is /home/zeppelin/prerequisites, although any location could be used.
#### Git
Intall latest stable version of Git. This document describes installation of version 2.4.8
```bash
yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel
yum install gcc perl-ExtUtils-MakeMaker
yum remove git
cd /home/zeppelin/prerequisites
wget https://github.com/git/git/archive/v2.4.8.tar.gz
tar xzf git-2.0.4.tar.gz
cd git-2.0.4
make prefix=/home/zeppelin/prerequisites/git all
make prefix=/home/zeppelin/prerequisites/git install
echo "export PATH=$PATH:/home/zeppelin/prerequisites/bin" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
git --version
```
Assuming all the packages are successfully installed, running the version option with git command should display
```bash
git version 2.4.8
```
#### Java
Zeppelin works well with 1.7.x version of Java runtime. Download JDK version 7 and a stable update and follow below instructions to install it.
```bash
cd /home/zeppelin/prerequisites/
#Download JDK 1.7, Assume JDK 7 update 79 is downloaded.
tar -xf jdk-7u79-linux-x64.tar.gz
echo "export JAVA_HOME=/home/zeppelin/prerequisites/jdk1.7.0_79" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
echo $JAVA_HOME
```
Assuming all the packages are successfully installed, echoing JAVA_HOME environment variable should display
```bash
/home/zeppelin/prerequisites/jdk1.7.0_79
```
#### Apache Maven
Download and install a stable version of Maven.
```bash
cd /home/zeppelin/prerequisites/
wget ftp://mirror.reverse.net/pub/apache/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
tar -xf apache-maven-3.3.3-bin.tar.gz
cd apache-maven-3.3.3
export MAVEN_HOME=/home/zeppelin/prerequisites/apache-maven-3.3.3
echo "export PATH=$PATH:/home/zeppelin/prerequisites/apache-maven-3.3.3/bin" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
mvn -version
```
Assuming all the packages are successfully installed, running the version option with mvn command should display
```bash
Apache Maven 3.3.3 (7994120775791599e205a5524ec3e0dfe41d4a06; 2015-04-22T04:57:37-07:00)
Maven home: /home/zeppelin/prerequisites/apache-maven-3.3.3
Java version: 1.7.0_79, vendor: Oracle Corporation
Java home: /home/zeppelin/prerequisites/jdk1.7.0_79/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "2.6.32-358.el6.x86_64", arch: "amd64", family: "unix"
```
It's assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine.
#### Hadoop client
Zeppelin can work with multiple versions & distributions of Hadoop. A complete list [is available here.](https://github.com/apache/incubator-zeppelin#build) This document assumes Hadoop 2.7.x client libraries including configuration files are installed on Zeppelin node. It also assumes /etc/hadoop/conf contains various Hadoop configuration files. The location of Hadoop configuration files may vary, hence use appropriate location.
Zeppelin can work with multiple versions & distributions of Hadoop. A complete list is available [here](https://github.com/apache/incubator-zeppelin#build). This document assumes Hadoop 2.7.x client libraries including configuration files are installed on Zeppelin node. It also assumes /etc/hadoop/conf contains various Hadoop configuration files. The location of Hadoop configuration files may vary, hence use appropriate location.
```bash
hadoop version
@ -134,32 +66,21 @@ This command was run using /usr/hdp/2.3.1.0-2574/hadoop/lib/hadoop-common-2.7.1.
```
#### Spark
Zeppelin can work with multiple versions Spark. A complete list [is available here.](https://github.com/apache/incubator-zeppelin#build) This document assumes Spark 1.3.1 is installed on Zeppelin node at /home/zeppelin/prerequisites/spark.
Spark is supported out of the box and to take advantage of this, you need to Download appropriate version of Spark binary packages from [Spark Download page](http://spark.apache.org/downloads.html) and unzip it.
Zeppelin can work with multiple versions of Spark. A complete list is available [here](https://github.com/apache/incubator-zeppelin#build).
This document assumes Spark 1.6.0 is installed at /usr/lib/spark.
> Note: Spark should be installed on the same node as Zeppelin.
## Build
> Note: Spark's pre-built package for CDH 4 doesn't support yarn.
Checkout source code from [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
#### Zeppelin
```bash
cd /home/zeppelin/
git clone https://github.com/apache/incubator-zeppelin.git
```
Zeppelin package is available at /home/zeppelin/incubator-zeppelin after the checkout completes.
### Cluster mode
As its assumed Hadoop 2.7.x is installed on the YARN cluster & Spark 1.3.1 is installed on Zeppelin node. Hence appropriate options are chosen to build Zeppelin. This is very important as Zeppelin will bundle corresponding Hadoop & Spark libraries and they must match the ones present on YARN cluster & Zeppelin Spark installation.
Zeppelin is a maven project and hence must be built with Apache Maven.
```bash
cd /home/zeppelin/incubator-zeppelin
mvn clean package -Pspark-1.3 -Dspark.version=1.3.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests
```
Building Zeppelin for first time downloads various dependencies and hence takes few minutes to complete.
Checkout source code from [git://git.apache.org/incubator-zeppelin.git](https://github.com/apache/incubator-zeppelin.git) or download binary package from [Download page](https://zeppelin.incubator.apache.org/download.html).
You can refer [Install](install.html) page for the details.
This document assumes that Zeppelin is located under `/home/zeppelin/incubator-zeppelin`.
## Zeppelin Configuration
Zeppelin configurations needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment XML
Zeppelin configuration needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment shell script.
```bash
cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh
@ -168,9 +89,10 @@ cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppeli
Set the following properties
```bash
export JAVA_HOME=/home/zeppelin/prerequisites/jdk1.7.0_79
export HADOOP_CONF_DIR=/etc/hadoop/conf
export JAVA_HOME="/usr/java/jdk1.7.0_79"
export HADOOP_CONF_DIR="/etc/hadoop/conf"
export ZEPPELIN_JAVA_OPTS="-Dhdp.version=2.3.1.0-2574"
export SPARK_HOME="/usr/lib/spark"
```
As /etc/hadoop/conf contains various configurations of YARN cluster, Zeppelin can now submit Spark/Hive jobs on YARN cluster form its web interface. The value of hdp.version is set to 2.3.1.0-2574. This can be obtained by running the following command
@ -196,7 +118,7 @@ bin/zeppelin-daemon.sh stop
```
## Interpreter
Zeppelin provides to various distributed processing frameworks to process data that ranges from Spark, Hive, Tajo, Ignite and Lens to name a few. This document describes to configure Hive & Spark interpreters.
Zeppelin provides various distributed processing frameworks to process data that ranges from Spark, Hive, Tajo, Ignite and Lens to name a few. This document describes to configure Hive & Spark interpreters.
### Hive
Zeppelin supports Hive interpreter and hence copy hive-site.xml that should be present at /etc/hive/conf to the configuration folder of Zeppelin. Once Zeppelin is built it will have conf folder under /home/zeppelin/incubator-zeppelin.
@ -209,7 +131,7 @@ Once Zeppelin server has started successfully, visit http://[zeppelin-server-hos
Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.
### Spark
Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spark is installed at /home/zeppelin/prerequisites/spark. Look for Spark configrations and click edit button to add the following properties
It was assumed that 1.6.0 version of Spark is installed at /usr/lib/spark. Look for Spark configurations and click edit button to add the following properties
<table class="table-configuration">
<tr>
@ -222,11 +144,6 @@ Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spa
<td>yarn-client</td>
<td>In yarn-client mode, the driver runs in the client process, and the application master is only used for requesting resources from YARN.</td>
</tr>
<tr>
<td>spark.home</td>
<td>/home/zeppelin/prerequisites/spark</td>
<td></td>
</tr>
<tr>
<td>spark.driver.extraJavaOptions</td>
<td>-Dhdp.version=2.3.1.0-2574</td>
@ -237,11 +154,6 @@ Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spa
<td>-Dhdp.version=2.3.1.0-2574</td>
<td></td>
</tr>
<tr>
<td>spark.yarn.jar</td>
<td>/home/zeppelin/incubator-zeppelin/interpreter/spark/zeppelin-spark-0.6.0-incubating-SNAPSHOT.jar</td>
<td></td>
</tr>
</table>
Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.

914
docs/interpreter/cassandra.md
View file

File diff suppressed because it is too large Load diff

228
docs/interpreter/jdbc.md Normal file
View file

@ -0,0 +1,228 @@
---
layout: page
title: "Generic JDBC Interpreter"
description: "JDBC user guide"
group: manual
---
{% include JB/setup %}
## Generic JDBC Interpreter for Apache Zeppelin
This interpreter lets you create a JDBC connection to any data source, by now it has been tested with:
* Postgres
* MySql
* MariaDB
* Redshift
* Hive
* Apache Drill
* Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver)
If someone else used another database please report how it works to improve functionality.
### Create Interpreter
When create a interpreter by default use PostgreSQL with the next properties:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:postgresql://localhost:5432/</td>
</tr>
<tr>
<td>default.user</td>
<td>gpadmin</td>
</tr>
</table>
It is not necessary to add driver jar to the classpath for PostgreSQL as it is included in Zeppelin.
#### Simple connection
Prior to creating the interpreter it is necessary to add maven coordinate or path of the JDBC driver to the Zeppelin classpath. To do this you must edit dependencies artifact(ex. `mysql:mysql-connector-java:5.1.38`) in interpreter menu as shown:
<div class="row">
<div class="col-md-11">
<img src="../assets/themes/zeppelin/img/docs-img/jdbc-simple-connection-setting.png" />
</div>
</div>
To create the interpreter you need to specify connection parameters as shown in the table.
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>driver name</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc url</td>
</tr>
<tr>
<td>default.user</td>
<td>user name</td>
</tr>
</table>
#### Multiple connections
JDBC interpreter also allows connections to multiple data sources. It is necessary to set a prefix for each connection to reference it in the paragraph in the form of `%jdbc(prefix)`. Before you create the interpreter it is necessary to add each driver's maven coordinates or JDBC driver's jar file path to the Zeppelin classpath. To do this you must edit the dependencies of JDBC interpreter in interpreter menu as following:
<div class="row">
<div class="col-md-11">
<img src="../assets/themes/zeppelin/img/docs-img/jdbc-multi-connection-setting.png" />
</div>
</div>
You can add all the jars you need to make multiple connections into the same JDBC interpreter. To create the interpreter you must specify the parameters. For example we will create two connections to MySQL and Redshift, the respective prefixes are `default` and `redshift`:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>com.mysql.jdbc.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:mysql://localhost:3306/</td>
</tr>
<tr>
<td>default.user</td>
<td>mysql-user</td>
</tr>
<tr>
<td>redshift.driver</td>
<td>com.amazon.redshift.jdbc4.Driver</td>
</tr>
<tr>
<td>redshift.password</td>
<td>********</td>
</tr>
<tr>
<td>redshift.url</td>
<td>jdbc:redshift://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439</td>
</tr>
<tr>
<td>redshift.user</td>
<td>redshift-user</td>
</tr>
</table>
### Bind to Notebook
In the `Notebook` click on the `settings` icon at the top-right corner. Use select/deselect to specify the interpreters to be used in the `Notebook`.
### More Properties
You can modify the interpreter configuration in the `Interpreter` section. The most common properties are as follows, but you can specify other properties that need to be connected.
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
</tr>
<tr>
<td>{prefix}.url</td>
<td>JDBC URL to connect, the URL must include the name of the database </td>
</tr>
<tr>
<td>{prefix}.user</td>
<td>JDBC user name</td>
</tr>
<tr>
<td>{prefix}.password</td>
<td>JDBC password</td>
</tr>
<tr>
<td>{prefix}.driver</td>
<td>JDBC driver name.</td>
</tr>
<tr>
<td>common.max_result</td>
<td>Max number of SQL result to display to prevent the browser overload. This is common properties for all connections</td>
</tr>
</table>
To develop this functionality use this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29). For example if a connection needs a schema parameter, it would have to add the property as follows:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>{prefix}.schema</td>
<td>schema_name</td>
</tr>
</table>
### How to use
#### Reference in paragraph
Start the paragraphs with the `%jdbc`, this will use the `default` prefix for connection. If you want to use other connection you should specify the prefix of it as follows `%jdbc(prefix)`:
```sql
%jdbc
SELECT * FROM db_name;
```
or
```sql
%jdbc(prefix)
SELECT * FROM db_name;
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
%jdbc(prefix)
SELECT name, country, performer
FROM demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
```
### Bugs & Contacts
If you find a bug for this interpreter, please create a [JIRA]( https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN) ticket.

29
docs/interpreter/spark.md
View file

@ -264,3 +264,32 @@ select * from ${table=defaultTableName} where text like '%${search}%'
```
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
### Separate Interpreter for each note
In 'Separate Interpreter for each note' mode, SparkInterpreter creates scala compiler per each notebook. However it still shares the single SparkContext.
## Setting up Zeppelin with Kerberos
Logical setup with Zeppelin, Kerberos Distribution Center (KDC), and Spark on YARN:
<img src="../assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png">
####Configuration Setup
1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf.
This is to make the server communicate with KDC.
2. Set SPARK\_HOME in [ZEPPELIN\_HOME]/conf/zeppelin-env.sh to use spark-submit
( Additionally, you might have to set “export HADOOP\_CONF\_DIR=/etc/hadoop/conf” )
3. Add the two properties below to spark configuration ( [SPARK_HOME]/conf/spark-defaults.conf ):
spark.yarn.principal
spark.yarn.keytab
> **NOTE:** If you do not have access to the above spark-defaults.conf file, optionally, you may add the lines to the Spark Interpreter through the Interpreter tab in the Zeppelin UI.
4. That's it. Play with Zeppelin !

33
docs/manual/dynamicform.md
View file

@ -54,6 +54,16 @@ Also you can separate option's display name and value, using _${formName=default
<img src="/assets/themes/zeppelin/img/screenshots/form_select_displayname.png" />
#### Checkbox form
For multi-selection, you can create a checkbox form using _${checkbox:formName=defaultValue1|defaultValue2...,option1|option2...}_. The variable will be substituted by a comma-separated string based on the selected items. For example:
<img src="/assets/themes/zeppelin/img/screenshots/form_checkbox.png">
Besides, you can specify the delimiter using _${checkbox(delimiter):formName=...}_:
<img src="/assets/themes/zeppelin/img/screenshots/form_checkbox_delimiter.png">
### Creates Programmatically
Some language backend uses programmatic way to create form. For example [ZeppelinContext](../interpreter/spark.html#zeppelincontext) provides form creation API
@ -134,3 +144,26 @@ print("Hello "+z.select("day", [("1","mon"),
</div>
</div>
<img src="/assets/themes/zeppelin/img/screenshots/form_select_prog.png" />
#### Checkbox form
<div class="codetabs">
<div data-lang="scala" markdown="1">
{% highlight scala %}
%spark
val options = Seq(("apple","Apple"), ("banana","Banana"), ("orange","Orange"))
println("Hello "+z.checkbox("fruit", options).mkString(" and "))
{% endhighlight %}
</div>
<div data-lang="python" markdown="1">
{% highlight python %}
%pyspark
options = [("apple","Apple"), ("banana","Banana"), ("orange","Orange")]
print("Hello "+ " and ".join(z.checkbox("fruit", options, ["apple"])))
{% endhighlight %}
</div>
</div>
<img src="/assets/themes/zeppelin/img/screenshots/form_checkbox_prog.png" />

17
docs/manual/interpreters.md
View file

@ -32,10 +32,16 @@ When you click the ```+Create``` button in the interpreter page, the interpreter
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
## What is Zeppelin Interpreter Setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
Each notebook can be binded to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
## What is Zeppelin Interpreter Group?
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, SparkSQL and the dependency loader.
@ -44,3 +50,12 @@ Technically, Zeppelin interpreters from the same group are running in the same J
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
## Interpreter binding mode
Each Interpreter Setting can choose one of two different interpreter binding mode.
Shared mode (default) and 'Separate Interpreter for each note' mode. In shared mode, every notebook binded to the Interpreter Setting will share the single Interpreter instance. In 'Separate Interpreter for each note' mode, each notebook will create new Interpreter instance. Therefore each notebook will have fresh new Interpreter environment.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">

48
docs/manual/publish.md Normal file
View file

@ -0,0 +1,48 @@
---
layout: page
title: "Publish your Paragraph"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## How can you publish your paragraph ?
Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website.
It's very straightforward. Just use `<iframe>` tag in your page.
> **Warning**: Please use this feature with caution and in a trusted environment only, as Zeppelin entire Webapp could be accessible for whoever visits your website.
### Copy a Paragraph Link
A first step to publish your paragraph result is **Copy a Paragraph Link**.
* After running a paragraph in your Zeppelin notebook, click a gear button located on the right side. Then, click **Link this Paragraph** menu like below image.
<center><img src="../assets/themes/zeppelin/img/docs-img/link-the-paragraph.png" height="100%" width="100%"></center>
* Just copy the provided link.
<center><img src="../assets/themes/zeppelin/img/docs-img/copy-the-link.png" height="100%" width="100%"></center>
### Embed the Paragraph to Your Website
For publishing the copied paragraph, you may use `<iframe>` tag in your website page.
For example,
```
<iframe src="http://< ip-address >:< port >/#/notebook/2B3QSZTKR/paragraph/...?asIframe" height="" width="" ></iframe>
```
Finally, you can show off your beautiful visualization results in your website.
<center><img src="../assets/themes/zeppelin/img/docs-img/your-website.png" height="90%" width="90%"></center>
> **Note**: To embed the paragraph in a website, Zeppelin needs to be reachable by that website.

72
docs/manual/shiroauthentication.md Normal file
View file

@ -0,0 +1,72 @@
---
layout: page
title: "Shiro Security for Apache Zeppelin"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Shiro authentication for Apache Zeppelin
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
When you connect to Apache Zeppelin, you will be asked to enter your credentials. Once you logged in, then you have access to all notes including other user's notes.
## Security Setup
You can setup **Zeppelin notebook authentication** in some simple steps.
####1. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
The default status of them is
```
/** = anon
#/** = authcBasic
```
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
```
#/** = anon
/** = authcBasic
```
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
####2. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
####3. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
```
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
```
admin = password1
user1 = password2
user2 = password3
```
Those combinations are defined in the `conf/shiro.ini` file.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/incubator-zeppelin/blob/master/SECURITY-README.md).

31
docs/security/authentication.md Normal file
View file

@ -0,0 +1,31 @@
---
layout: page
title: "Authentication"
description: "Authentication"
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Authentication
Authentication is company-specific.
One option is to use [Basic Access Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication)
Another option is to have an authentication server that can verify user credentials in an LDAP server.
If an incoming request to the Zeppelin server does not have a cookie with user information encrypted with the authentication server public key, the user
is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific
URL in the Zeppelin server which sets the authentication cookie in the browser.
The end result is that all requests to the Zeppelin
web server have the authentication cookie which contains user and groups information.

34
docs/security/interpreter_authorization.md Normal file
View file

@ -0,0 +1,34 @@
---
layout: page
title: "Notebook Authorization"
description: "Notebook Authorization"
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Interpreter and Data Source Authorization
## Interpreter Authorization
Interpreter authorization involves permissions like creating an interpreter and execution queries using it.
## Data Source Authorization
Data source authorization involves authenticating to the data source like a Mysql database and letting it determine user permissions.
For the Hive interpreter, we need to maintain per-user connection pools.
The interpret method takes the user string as parameter and executes the jdbc call using a connection in the user's connection pool.
In case of Presto, we don't need password if the Presto DB server runs backend code using HDFS authorization for the user.
For databases like Vertica and Mysql we have to store password information for users.

37
docs/security/notebook_authorization.md Normal file
View file

@ -0,0 +1,37 @@
---
layout: page
title: "Notebook Authorization"
description: "Notebook Authorization"
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Notebook Authorization
We assume that there is an authentication component that associates a user string and a set of group strings with every NotebookSocket.
Each note has the following:
* set of owner entities (users or groups)
* set of reader entities (users or groups)
* set of writer entities (users or groups)
If a set is empty, it means that any user can perform that operation.
The NotebookServer classifies every Note operation into three categories: read, write, manage.
Before executing a Note operation, it checks if the user and the groups associated with the NotebookSocket have permissions. For example, before executing an read
operation, it checks if the user and the groups have at least one entity that belongs to the reader entities.
To initialize and modify note permissions, we provide UI like "Interpreter binding". The user inputs comma separated entities for owners, readers and writers.
We execute a rest api call with this information. In the backend we get the user information for the connection and allow the operation if the user and groups
associated with the current user have at least one entity that belongs to owner entities for the note.

28
docs/security/overview.md Normal file
View file

@ -0,0 +1,28 @@
---
layout: page
title: "Security Overview"
description: "Security Overview"
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Security Overview
There are three aspects to Zeppelin security:
* Authentication: is the user who they say they are? [More](authentication.html)
* Notebook authorization: does the user have permissions to read or write to a note? [More](notebook_authorization.html)
* Interpreter and data source authorization: does the user have permissions to perform interpreter operations or access data source objects? [More](interpreter_authorization.html)

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class FlinkInterpreterTest {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null);
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
}
@AfterClass

37
hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java
View file

@ -169,6 +169,9 @@ public class HiveInterpreter extends Interpreter {
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
}
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
@ -203,6 +206,10 @@ public class HiveInterpreter extends Interpreter {
} else {
connection = getConnection(propertyKey);
}
if (connection == null) {
return null;
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
@ -232,6 +239,10 @@ public class HiveInterpreter extends Interpreter {
Statement statement = getStatement(propertyKey, paragraphId);
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
statement.setMaxRows(getMaxResult());
StringBuilder msg;
@ -315,10 +326,8 @@ public class HiveInterpreter extends Interpreter {
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
if (null != propertyKey && !propertyKey.equals(DEFAULT_KEY)) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
@ -334,17 +343,19 @@ public class HiveInterpreter extends Interpreter {
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
boolean firstLineIndex = cmd.startsWith("(");
if (firstLineIndex) {
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1) {
return cmd.substring(configStartIndex + 1, configLastIndex);
} else {
return null;
}
} else {
return DEFAULT_KEY;
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override

58
hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java
View file

@ -66,7 +66,53 @@ public class HiveInterpreterTest {
@After
public void tearDown() throws Exception {
}
@Test
public void testForParsePropertyKey() throws IOException {
HiveInterpreter t = new HiveInterpreter(new Properties());
assertEquals(t.getPropertyKey("(fake) select max(cant) from test_table where id >= 2452640"),
"fake");
assertEquals(t.getPropertyKey("() select max(cant) from test_table where id >= 2452640"),
"");
assertEquals(t.getPropertyKey(")fake( select max(cant) from test_table where id >= 2452640"),
"default");
// when you use a %hive(prefix1), prefix1 is the propertyKey as form part of the cmd string
assertEquals(t.getPropertyKey("(prefix1)\n select max(cant) from test_table where id >= 2452640"),
"prefix1");
assertEquals(t.getPropertyKey("(prefix2) select max(cant) from test_table where id >= 2452640"),
"prefix2");
// when you use a %hive, prefix is the default
assertEquals(t.getPropertyKey("select max(cant) from test_table where id >= 2452640"),
"default");
}
@Test
public void testForMapPrefix() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
String sqlQuery = "(fake) select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
// if prefix not found return ERROR and Prefix not found.
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals("Prefix not found.", interpreterResult.message());
}
@Test
public void readTest() throws IOException {
Properties properties = new Properties();
@ -79,9 +125,9 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null,null,null)).message().contains("SCHEMA_NAME"));
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message().contains("SCHEMA_NAME"));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null,null)).message());
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message());
}
@Test
@ -101,7 +147,7 @@ public class HiveInterpreterTest {
t.open();
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null,null)).message());
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message());
}
@Test
@ -117,13 +163,13 @@ public class HiveInterpreterTest {
t.open();
InterpreterResult interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
t.getConnection("default").close();
interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@ -139,7 +185,7 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null, null);
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null, null, null);
//simple select test
InterpreterResult result = t.interpret("select * from test_table", interpreterContext);

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -44,7 +44,7 @@ public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;

50
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@ -172,6 +173,9 @@ public class JDBCInterpreter extends Interpreter {
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
}
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
@ -206,6 +210,10 @@ public class JDBCInterpreter extends Interpreter {
} else {
connection = getConnection(propertyKey);
}
if (connection == null) {
return null;
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
@ -260,6 +268,10 @@ public class JDBCInterpreter extends Interpreter {
try {
Statement statement = getStatement(propertyKey, paragraphId);
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
statement.setMaxRows(getMaxResult());
StringBuilder msg = null;
@ -293,7 +305,15 @@ public class JDBCInterpreter extends Interpreter {
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
msg.append(replaceReservedChars(isTableType, resultSet.getString(i)));
Object resultObject;
String resultValue;
resultObject = resultSet.getObject(i);
if (resultObject == null) {
resultValue = "null";
} else {
resultValue = resultSet.getString(i);
}
msg.append(replaceReservedChars(isTableType, resultValue));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
@ -344,12 +364,10 @@ public class JDBCInterpreter extends Interpreter {
logger.info("Run SQL command '{}'", cmd);
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
if (null != propertyKey && !propertyKey.equals(DEFAULT_KEY)) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
@ -371,17 +389,19 @@ public class JDBCInterpreter extends Interpreter {
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
boolean firstLineIndex = cmd.startsWith("(");
if (firstLineIndex) {
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1) {
return cmd.substring(configStartIndex + 1, configLastIndex);
} else {
return null;
}
} else {
return DEFAULT_KEY;
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override

87
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -26,10 +26,7 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.*;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -64,11 +61,59 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
statement.execute(
"DROP TABLE IF EXISTS test_table; " +
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
statement.execute(
"insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name');"
);
PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
}
@Test
public void testForParsePropertyKey() throws IOException {
JDBCInterpreter t = new JDBCInterpreter(new Properties());
assertEquals(t.getPropertyKey("(fake) select max(cant) from test_table where id >= 2452640"),
"fake");
assertEquals(t.getPropertyKey("() select max(cant) from test_table where id >= 2452640"),
"");
assertEquals(t.getPropertyKey(")fake( select max(cant) from test_table where id >= 2452640"),
"default");
// when you use a %jdbc(prefix1), prefix1 is the propertyKey as form part of the cmd string
assertEquals(t.getPropertyKey("(prefix1)\n select max(cant) from test_table where id >= 2452640"),
"prefix1");
assertEquals(t.getPropertyKey("(prefix2) select max(cant) from test_table where id >= 2452640"),
"prefix2");
// when you use a %jdbc, prefix is the default
assertEquals(t.getPropertyKey("select max(cant) from test_table where id >= 2452640"),
"default");
}
@Test
public void testForMapPrefix() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "(fake) select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
// if prefix not found return ERROR and Prefix not found.
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals("Prefix not found.", interpreterResult.message());
}
@Test
public void testDefaultProperties() throws SQLException {
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(new Properties());
@ -92,15 +137,37 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table";
String sqlQuery = "select * from test_table WHERE ID in ('a', 'b')";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@Test
public void testSelectQueryWithNull() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table WHERE ID = 'c'";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\nc\tnull\n", interpreterResult.message());
}
@Test
public void testSelectQueryMaxResult() throws SQLException, IOException {
@ -116,7 +183,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());

11
lens/pom.xml
View file

@ -226,15 +226,4 @@
</plugins>
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
</profiles>
</project>

11
pom.xml
View file

@ -446,6 +446,7 @@
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>**/*.diff</exclude>
@ -635,6 +636,16 @@
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<!-- Geode can be enabled by -Pgeode. see https://issues.apache.org/jira/browse/ZEPPELIN-375 -->
<profile>
<id>geode</id>

3
scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java
View file

@ -26,6 +26,7 @@ import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@ -62,7 +63,7 @@ public class ScaldingInterpreterTest {
}
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), null);

14
scripts/vagrant/zeppelin-dev/README.md
View file

@ -24,9 +24,9 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
@ -45,7 +45,7 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
@ -53,7 +53,7 @@ Cloning the project again may seem counter intuitive, since this script likley o
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -61,7 +61,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -104,7 +104,7 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..

2
scripts/vagrant/zeppelin-dev/show-instructions.sh
View file

@ -18,7 +18,7 @@ echo '# Post vagrant up instructions.'
echo '# From your host machine,'
echo '# git clone the incubator-zeppelin branch into this directory'
echo
echo 'git clone https://github.com/apache/incubator-zeppelin.git'
echo 'git clone git://git.apache.org/incubator-zeppelin.git'
echo
echo '# Cloning the project again may seem counter intuitive, since this script'
echo '# originated from the project repository. Consider copying just the vagrant/zeppelin-dev'

29
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -32,6 +32,7 @@ import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Job;
@ -46,10 +47,22 @@ import org.slf4j.LoggerFactory;
public class ShellInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(ShellInterpreter.class);
private static final String EXECUTOR_KEY = "executor";
int commandTimeOut = 600000;
public static final String SHELL_COMMAND_TIMEOUT = "shell.command.timeout.millisecs";
public static final String DEFAULT_COMMAND_TIMEOUT = "600000";
int commandTimeOut;
static {
Interpreter.register("sh", ShellInterpreter.class.getName());
Interpreter.register(
"sh",
"sh",
ShellInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(
SHELL_COMMAND_TIMEOUT,
DEFAULT_COMMAND_TIMEOUT,
"Shell command time out in millisecs. Default = 600000")
.build()
);
}
public ShellInterpreter(Properties property) {
@ -57,7 +70,11 @@ public class ShellInterpreter extends Interpreter {
}
@Override
public void open() {}
public void open() {
logger.info("Command timeout is set as:", SHELL_COMMAND_TIMEOUT);
commandTimeOut = Integer.valueOf(getProperty(SHELL_COMMAND_TIMEOUT));
}
@Override
public void close() {}
@ -72,7 +89,7 @@ public class ShellInterpreter extends Interpreter {
DefaultExecutor executor = new DefaultExecutor();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream));
executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeOut));
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
@ -82,7 +99,7 @@ public class ShellInterpreter extends Interpreter {
int exitVal = executor.execute(cmdLine);
logger.info("Paragraph " + contextInterpreter.getParagraphId()
+ "return with exit value: " + exitVal);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString());
return new InterpreterResult(InterpreterResult.Code.SUCCESS, null);
} catch (ExecuteException e) {
int exitValue = e.getExitValue();
logger.error("Can not run " + cmd, e);
@ -94,7 +111,7 @@ public class ShellInterpreter extends Interpreter {
logger.info("The paragraph " + contextInterpreter.getParagraphId()
+ " stopped executing: " + msg);
}
msg += "Exitvalue: " + exitValue;
msg += "ExitValue: " + exitValue;
return new InterpreterResult(code, msg);
} catch (IOException e) {
logger.error("Can not run " + cmd, e);

12
spark-dependencies/pom.xml
View file

@ -339,16 +339,6 @@
</dependencies>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<profile>
<id>spark-1.1</id>
<dependencies>
@ -501,7 +491,7 @@
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.5.0-RC1</version>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>

28
spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java
View file

@ -299,23 +299,25 @@ public class DepInterpreter extends Interpreter {
if (intpGroup == null) {
return null;
}
synchronized (intpGroup) {
for (Interpreter intp : intpGroup){
if (intp.getClassName().equals(SparkInterpreter.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (SparkInterpreter) p;
}
}
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
if (p == null) {
return null;
}
return null;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (SparkInterpreter) p;
}
@Override
public Scheduler getScheduler() {
return getSparkInterpreter().getScheduler();
SparkInterpreter sparkInterpreter = getSparkInterpreter();
if (sparkInterpreter != null) {
return getSparkInterpreter().getScheduler();
} else {
return null;
}
}
}

42
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -494,23 +494,18 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
private SparkInterpreter getSparkInterpreter() {
InterpreterGroup intpGroup = getInterpreterGroup();
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
synchronized (intpGroup) {
for (Interpreter intp : getInterpreterGroup()){
if (intp.getClassName().equals(SparkInterpreter.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
spark = (SparkInterpreter) p;
}
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
spark = (SparkInterpreter) p;
if (lazy != null) {
lazy.open();
}
@ -554,20 +549,15 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
private DepInterpreter getDepInterpreter() {
InterpreterGroup intpGroup = getInterpreterGroup();
if (intpGroup == null) return null;
synchronized (intpGroup) {
for (Interpreter intp : intpGroup) {
if (intp.getClassName().equals(DepInterpreter.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (DepInterpreter) p;
}
}
Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
if (p == null) {
return null;
}
return null;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (DepInterpreter) p;
}

173
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -20,11 +20,13 @@ package org.apache.zeppelin.spark;
import java.io.File;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Joiner;
@ -44,7 +46,6 @@ import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@ -57,17 +58,15 @@ import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Console;
import scala.*;
import scala.Enumeration.Value;
import scala.None;
import scala.Some;
import scala.Tuple2;
import scala.collection.Iterator;
import scala.collection.JavaConversions;
import scala.collection.JavaConverters;
import scala.collection.Seq;
import scala.collection.mutable.HashMap;
import scala.collection.mutable.HashSet;
import scala.reflect.io.AbstractFile;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.Completion.Candidates;
import scala.tools.nsc.interpreter.Completion.ScalaCompleter;
@ -113,16 +112,19 @@ public class SparkInterpreter extends Interpreter {
private ZeppelinContext z;
private SparkILoop interpreter;
private SparkIMain intp;
private SparkContext sc;
private static SparkContext sc;
private static SQLContext sqlc;
private static SparkEnv env;
private static JobProgressListener sparkListener;
private static AbstractFile classOutputDir;
private static Integer sharedInterpreterLock = new Integer(0);
private static AtomicInteger numReferenceOfSparkContext = new AtomicInteger(0);
private SparkOutputStream out;
private SQLContext sqlc;
private SparkDependencyResolver dep;
private SparkJLineCompletion completor;
private JobProgressListener sparkListener;
private Map<String, Object> binder;
private SparkEnv env;
private SparkVersion sparkVersion;
@ -139,17 +141,21 @@ public class SparkInterpreter extends Interpreter {
sparkListener = setupListeners(this.sc);
}
public synchronized SparkContext getSparkContext() {
if (sc == null) {
sc = createSparkContext();
env = SparkEnv.get();
sparkListener = setupListeners(sc);
public SparkContext getSparkContext() {
synchronized (sharedInterpreterLock) {
if (sc == null) {
sc = createSparkContext();
env = SparkEnv.get();
sparkListener = setupListeners(sc);
}
return sc;
}
return sc;
}
public boolean isSparkContextInitialized() {
return sc != null;
synchronized (sharedInterpreterLock) {
return sc != null;
}
}
static JobProgressListener setupListeners(SparkContext context) {
@ -192,33 +198,34 @@ public class SparkInterpreter extends Interpreter {
}
private boolean useHiveContext() {
return Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext"));
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext"));
}
public SQLContext getSQLContext() {
if (sqlc == null) {
if (useHiveContext()) {
String name = "org.apache.spark.sql.hive.HiveContext";
Constructor<?> hc;
try {
hc = getClass().getClassLoader().loadClass(name)
.getConstructor(SparkContext.class);
sqlc = (SQLContext) hc.newInstance(getSparkContext());
} catch (NoSuchMethodException | SecurityException
| ClassNotFoundException | InstantiationException
| IllegalAccessException | IllegalArgumentException
| InvocationTargetException e) {
logger.warn("Can't create HiveContext. Fallback to SQLContext", e);
// when hive dependency is not loaded, it'll fail.
// in this case SQLContext can be used.
synchronized (sharedInterpreterLock) {
if (sqlc == null) {
if (useHiveContext()) {
String name = "org.apache.spark.sql.hive.HiveContext";
Constructor<?> hc;
try {
hc = getClass().getClassLoader().loadClass(name)
.getConstructor(SparkContext.class);
sqlc = (SQLContext) hc.newInstance(getSparkContext());
} catch (NoSuchMethodException | SecurityException
| ClassNotFoundException | InstantiationException
| IllegalAccessException | IllegalArgumentException
| InvocationTargetException e) {
logger.warn("Can't create HiveContext. Fallback to SQLContext", e);
// when hive dependency is not loaded, it'll fail.
// in this case SQLContext can be used.
sqlc = new SQLContext(getSparkContext());
}
} else {
sqlc = new SQLContext(getSparkContext());
}
} else {
sqlc = new SQLContext(getSparkContext());
}
return sqlc;
}
return sqlc;
}
public SparkDependencyResolver getDependencyResolver() {
@ -232,20 +239,15 @@ public class SparkInterpreter extends Interpreter {
}
private DepInterpreter getDepInterpreter() {
InterpreterGroup intpGroup = getInterpreterGroup();
if (intpGroup == null) return null;
synchronized (intpGroup) {
for (Interpreter intp : intpGroup) {
if (intp.getClassName().equals(DepInterpreter.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (DepInterpreter) p;
}
}
Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName());
if (p == null) {
return null;
}
return null;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
return (DepInterpreter) p;
}
public SparkContext createSparkContext() {
@ -326,15 +328,24 @@ public class SparkInterpreter extends Interpreter {
}
}
pythonLibUris.trimToSize();
if (pythonLibs.length == pythonLibUris.size()) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
// Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
// when spark version is less than or equal to 1.4.1
if (pythonLibUris.size() == 2) {
try {
String confValue = conf.get("spark.yarn.dist.files");
conf.set("spark.yarn.dist.files", confValue + "," + Joiner.on(",").join(pythonLibUris));
} catch (NoSuchElementException e) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
}
if (!useSparkSubmit()) {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
}
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
}
// Distributes needed libraries to workers.
// Distributes needed libraries to workers
// when spark version is greater than or equal to 1.5.0
if (getProperty("master").equals("yarn-client")) {
conf.set("spark.yarn.isPython", "true");
}
@ -468,7 +479,9 @@ public class SparkInterpreter extends Interpreter {
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
/* spark interpreter */
System.setProperty("scala.repl.name.line", "line" + this.hashCode() + "$");
/* create scala repl */
this.interpreter = new SparkILoop(null, new PrintWriter(out));
interpreter.settings_$eq(settings);
@ -479,21 +492,39 @@ public class SparkInterpreter extends Interpreter {
intp.setContextClassLoader();
intp.initializeSynchronous();
completor = new SparkJLineCompletion(intp);
synchronized (sharedInterpreterLock) {
if (classOutputDir == null) {
classOutputDir = settings.outputDirs().getSingleOutput().get();
} else {
// change SparkIMain class output dir
settings.outputDirs().setSingleOutput(classOutputDir);
ClassLoader cl = intp.classLoader();
sc = getSparkContext();
if (sc.getPoolForName("fair").isEmpty()) {
Value schedulingMode = org.apache.spark.scheduler.SchedulingMode.FAIR();
int minimumShare = 0;
int weight = 1;
Pool pool = new Pool("fair", schedulingMode, minimumShare, weight);
sc.taskScheduler().rootPool().addSchedulable(pool);
try {
Field rootField = cl.getClass().getSuperclass().getDeclaredField("root");
rootField.setAccessible(true);
rootField.set(cl, classOutputDir);
} catch (NoSuchFieldException | IllegalAccessException e) {
logger.error(e.getMessage(), e);
}
}
completor = new SparkJLineCompletion(intp);
sc = getSparkContext();
if (sc.getPoolForName("fair").isEmpty()) {
Value schedulingMode = org.apache.spark.scheduler.SchedulingMode.FAIR();
int minimumShare = 0;
int weight = 1;
Pool pool = new Pool("fair", schedulingMode, minimumShare, weight);
sc.taskScheduler().rootPool().addSchedulable(pool);
}
sparkVersion = SparkVersion.fromVersionString(sc.version());
sqlc = getSQLContext();
}
sparkVersion = SparkVersion.fromVersionString(sc.version());
sqlc = getSQLContext();
dep = getDependencyResolver();
z = new ZeppelinContext(sc, sqlc, null, dep,
@ -585,6 +616,8 @@ public class SparkInterpreter extends Interpreter {
}
}
}
numReferenceOfSparkContext.incrementAndGet();
}
private List<File> currentClassPath() {
@ -907,8 +940,12 @@ public class SparkInterpreter extends Interpreter {
@Override
public void close() {
sc.stop();
sc = null;
logger.info("Close interpreter");
if (numReferenceOfSparkContext.decrementAndGet() == 0) {
sc.stop();
sc = null;
}
intp.close();
}

36
spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
View file

@ -79,23 +79,18 @@ public class SparkSqlInterpreter extends Interpreter {
}
private SparkInterpreter getSparkInterpreter() {
InterpreterGroup intpGroup = getInterpreterGroup();
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
synchronized (intpGroup) {
for (Interpreter intp : getInterpreterGroup()){
if (intp.getClassName().equals(SparkInterpreter.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
spark = (SparkInterpreter) p;
}
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
spark = (SparkInterpreter) p;
if (lazy != null) {
lazy.open();
}
@ -179,15 +174,14 @@ public class SparkSqlInterpreter extends Interpreter {
// It's because of scheduler is not created yet, and scheduler is created by this function.
// Therefore, we can still use getSparkInterpreter() here, but it's better and safe
// to getSparkInterpreter without opening it.
for (Interpreter intp : getInterpreterGroup()) {
if (intp.getClassName().equals(SparkInterpreter.class.getName())) {
Interpreter p = intp;
return p.getScheduler();
} else {
continue;
}
Interpreter intp =
getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
if (intp != null) {
return intp.getScheduler();
} else {
return null;
}
throw new InterpreterException("Can't find SparkInterpreter");
}
}

25
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -17,7 +17,9 @@
package org.apache.zeppelin.spark;
import static scala.collection.JavaConversions.asJavaCollection;
import static scala.collection.JavaConversions.asJavaIterable;
import static scala.collection.JavaConversions.asScalaIterable;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@ -84,6 +86,27 @@ public class ZeppelinContext {
public Object select(String name, Object defaultValue,
scala.collection.Iterable<Tuple2<Object, String>> options) {
return gui.select(name, defaultValue, tuplesToParamOptions(options));
}
public scala.collection.Iterable<Object> checkbox(String name,
scala.collection.Iterable<Tuple2<Object, String>> options) {
List<Object> allChecked = new LinkedList<Object>();
for (Tuple2<Object, String> option : asJavaIterable(options)) {
allChecked.add(option._1());
}
return checkbox(name, asScalaIterable(allChecked), options);
}
public scala.collection.Iterable<Object> checkbox(String name,
scala.collection.Iterable<Object> defaultChecked,
scala.collection.Iterable<Tuple2<Object, String>> options) {
return asScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked),
tuplesToParamOptions(options)));
}
private ParamOption[] tuplesToParamOptions(
scala.collection.Iterable<Tuple2<Object, String>> options) {
int n = options.size();
ParamOption[] paramOptions = new ParamOption[n];
Iterator<Tuple2<Object, String>> it = asJavaIterable(options).iterator();
@ -94,7 +117,7 @@ public class ZeppelinContext {
paramOptions[i++] = new ParamOption(valueAndDisplayValue._1(), valueAndDisplayValue._2());
}
return gui.select(name, defaultValue, paramOptions);
return paramOptions;
}
public void setGui(GUI o) {

10
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -84,6 +84,16 @@ class PyZeppelinContext(dict):
iterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(tuples)
return self.z.select(name, defaultValue, iterables)
def checkbox(self, name, options, defaultChecked = None):
if defaultChecked is None:
defaultChecked = map(lambda items: items[0], options)
optionTuples = map(lambda items: self.__tupleToScalaTuple2(items), options)
optionIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(optionTuples)
defaultCheckedIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(defaultChecked)
checkedIterables = self.z.checkbox(name, defaultCheckedIterables, optionIterables)
return gateway.jvm.scala.collection.JavaConversions.asJavaCollection(checkedIterables)
def __tupleToScalaTuple2(self, tuple):
if (len(tuple) == 2):
return gateway.jvm.scala.Tuple2(tuple[0], tuple[1])

14
spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java
View file

@ -25,11 +25,9 @@ import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.After;
import org.junit.Before;
@ -54,11 +52,13 @@ public class DepInterpreterTest {
dep.open();
InterpreterGroup intpGroup = new InterpreterGroup();
intpGroup.add(new SparkInterpreter(p));
intpGroup.add(dep);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(new SparkInterpreter(p));
intpGroup.get("note").add(dep);
dep.setInterpreterGroup(intpGroup);
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(), null);

55
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -24,9 +24,12 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import org.apache.spark.HttpServer;
import org.apache.spark.SecurityManager;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@ -41,11 +44,11 @@ import org.slf4j.LoggerFactory;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class SparkInterpreterTest {
public static SparkInterpreter repl;
public static InterpreterGroup intpGroup;
private InterpreterContext context;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterTest.class);
/**
* Get spark version number as a numerical value.
* eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ...
@ -69,29 +72,32 @@ public class SparkInterpreterTest {
if (repl == null) {
Properties p = new Properties();
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
repl = new SparkInterpreter(p);
repl.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl);
repl.open();
}
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
}));
}
@After
@ -186,4 +192,21 @@ public class SparkInterpreterTest {
}
}
}
@Test
public void shareSingleSparkContext() throws InterruptedException {
// create another SparkInterpreter
Properties p = new Properties();
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
assertEquals(Code.SUCCESS,
repl.interpret("print(sc.parallelize(1 to 10).count())", context).code());
assertEquals(Code.SUCCESS,
repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code());
repl2.close();
}
}

18
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -24,10 +24,10 @@ import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -51,21 +51,27 @@ public class SparkSqlInterpreterTest {
if (SparkInterpreterTest.repl == null) {
repl = new SparkInterpreter(p);
intpGroup = new InterpreterGroup();
repl.setInterpreterGroup(intpGroup);
repl.open();
SparkInterpreterTest.repl = repl;
SparkInterpreterTest.intpGroup = intpGroup;
} else {
repl = SparkInterpreterTest.repl;
intpGroup = SparkInterpreterTest.intpGroup;
}
sql = new SparkSqlInterpreter(p);
sql = new SparkSqlInterpreter(p);
intpGroup = new InterpreterGroup();
intpGroup.add(repl);
intpGroup.add(sql);
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(repl);
intpGroup.get("note").add(sql);
sql.setInterpreterGroup(intpGroup);
sql.open();
}
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(new InterpreterOutputListener() {

70
testing/downloadSpark.sh Executable file
View file

@ -0,0 +1,70 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
if [ $# -ne 2 ]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
fi
else
SPARK_VER_RANGE=">1.3"
fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
echo "SPARK_HOME is ${SPARK_HOME}"
if [ ! -d "${SPARK_HOME}" ]; then
if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
STARTTIME=`date +%s`
timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
STARTTIME=`date +%s`
timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}"
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
set +xe

40
testing/startSparkCluster.sh
View file

@ -26,41 +26,41 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
fi
else
SPARK_VER_RANGE=">1.3"
fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
echo "SPARK_HOME is ${SPARK_HOME} "
if [ ! -d "${SPARK_HOME}" ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
wget -q "${PREFFERED}${PATHINFO}"
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
echo "SPARK_HOME is ${SPARK_HOME}"
# create PID dir. test case detect pid file so they can select active spark home dir for test
mkdir -p ${SPARK_HOME}/run
export SPARK_PID_DIR=${SPARK_HOME}/run
# start
export SPARK_MASTER_PORT=7071
export SPARK_MASTER_WEBUI_PORT=7072
export SPARK_WORKER_WEBUI_PORT=8082
${SPARK_HOME}/sbin/start-master.sh
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then # spark 1.3 or prior
if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
# spark 1.3 or prior
${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
else
${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
fi
set +xe

4
testing/stopSparkCluster.sh
View file

@ -25,6 +25,8 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
@ -32,6 +34,6 @@ export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VER
# set create PID dir
export SPARK_PID_DIR=${SPARK_HOME}/run
${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
${SPARK_HOME}/sbin/stop-master.sh
set +xe

5
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularElemTest.scala
View file

@ -21,6 +21,7 @@ import java.util
import org.apache.zeppelin.display.{AngularObject, AngularObjectRegistry, GUI}
import org.apache.zeppelin.interpreter._
import org.apache.zeppelin.user.AuthenticationInfo
import org.scalatest.concurrent.Eventually
import org.scalatest.{BeforeAndAfter, BeforeAndAfterEach, FlatSpec, Matchers}
@ -33,8 +34,8 @@ trait AbstractAngularElemTest
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "paragraph", "title", "text",
new util.HashMap[String, Object](), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
new AuthenticationInfo(), new util.HashMap[String, Object](), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new util.LinkedList[InterpreterContextRunner](),
new InterpreterOutput(new InterpreterOutputListener() {

3
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularModelTest.scala
View file

@ -18,6 +18,7 @@ package org.apache.zeppelin.display.angular
import org.apache.zeppelin.display.{AngularObjectRegistry, GUI}
import org.apache.zeppelin.interpreter._
import org.apache.zeppelin.user.AuthenticationInfo
import org.scalatest.concurrent.Eventually
import org.scalatest.{BeforeAndAfter, BeforeAndAfterEach, FlatSpec, Matchers}
@ -28,7 +29,7 @@ trait AbstractAngularModelTest extends FlatSpec
with BeforeAndAfter with BeforeAndAfterEach with Eventually with Matchers {
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "id", "title", "text",
val context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new java.util.HashMap[String, Object](), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
null,

1
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -100,6 +100,7 @@ The following components are provided under Apache License.
(Apache 2.0) Tachyon Servers (org.tachyonproject:tachyon-servers:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Minicluster (org.tachyonproject:tachyon-minicluster:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Underfs Local (org.tachyonproject:tachyon-underfs-local:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Microsoft Azure Storage Library for Java (com.microsoft.azure:azure-storage:4.0.0 - https://github.com/Azure/azure-storage-java)

202
zeppelin-distribution/src/bin_license/licenses/LICENSE-azure-storage-4.0.0 Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

40
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
View file

@ -71,8 +71,8 @@ public class DependencyResolver extends AbstractDependencyResolver {
public synchronized List<File> load(String artifact, Collection<String> excludes)
throws RepositoryException, IOException {
if (StringUtils.isBlank(artifact)) {
// Should throw here
throw new RuntimeException("Invalid artifact to load");
// Skip dependency loading if artifact is empty
return new LinkedList<File>();
}
// <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>
@ -88,22 +88,26 @@ public class DependencyResolver extends AbstractDependencyResolver {
public List<File> load(String artifact, Collection<String> excludes, String destPath)
throws RepositoryException, IOException {
List<File> libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile: libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
List<File> libs = new LinkedList<File>();
if (StringUtils.isNotBlank(artifact)) {
libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile : libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}
}
return libs;

32
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/GUI.java
View file

@ -18,7 +18,10 @@
package org.apache.zeppelin.display;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
@ -59,7 +62,7 @@ public class GUI implements Serializable {
value = defaultValue;
}
forms.put(id, new Input(id, defaultValue));
forms.put(id, new Input(id, defaultValue, "input"));
return value;
}
@ -72,10 +75,35 @@ public class GUI implements Serializable {
if (value == null) {
value = defaultValue;
}
forms.put(id, new Input(id, defaultValue, options));
forms.put(id, new Input(id, defaultValue, "select", options));
return value;
}
public Collection<Object> checkbox(String id, Collection<Object> defaultChecked,
ParamOption[] options) {
Collection<Object> checked = (Collection<Object>) params.get(id);
if (checked == null) {
checked = defaultChecked;
}
forms.put(id, new Input(id, defaultChecked, "checkbox", options));
Collection<Object> filtered = new LinkedList<Object>();
for (Object o : checked) {
if (isValidOption(o, options)) {
filtered.add(o);
}
}
return filtered;
}
private boolean isValidOption(Object o, ParamOption[] options) {
for (ParamOption option : options) {
if (o.equals(option.getValue())) {
return true;
}
}
return false;
}
public void clear() {
this.forms = new TreeMap<String, Input>();
}

277
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java
View file

@ -17,8 +17,12 @@
package org.apache.zeppelin.display;
import org.apache.commons.lang.StringUtils;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -43,6 +47,25 @@ public class Input implements Serializable {
this.displayName = displayName;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ParamOption that = (ParamOption) o;
if (value != null ? !value.equals(that.value) : that.value != null) return false;
return displayName != null ? displayName.equals(that.displayName) : that.displayName == null;
}
@Override
public int hashCode() {
int result = value != null ? value.hashCode() : 0;
result = 31 * result + (displayName != null ? displayName.hashCode() : 0);
return result;
}
public Object getValue() {
return value;
}
@ -64,29 +87,32 @@ public class Input implements Serializable {
String name;
String displayName;
String type;
String argument;
Object defaultValue;
ParamOption[] options;
boolean hidden;
public Input(String name, Object defaultValue) {
public Input(String name, Object defaultValue, String type) {
this.name = name;
this.displayName = name;
this.defaultValue = defaultValue;
this.type = type;
}
public Input(String name, Object defaultValue, ParamOption[] options) {
public Input(String name, Object defaultValue, String type, ParamOption[] options) {
this.name = name;
this.displayName = name;
this.defaultValue = defaultValue;
this.type = type;
this.options = options;
}
public Input(String name, String displayName, String type, Object defaultValue,
public Input(String name, String displayName, String type, String argument, Object defaultValue,
ParamOption[] options, boolean hidden) {
super();
this.name = name;
this.displayName = displayName;
this.argument = argument;
this.type = type;
this.defaultValue = defaultValue;
this.options = options;
@ -142,6 +168,18 @@ public class Input implements Serializable {
return hidden;
}
// Syntax of variables: ${TYPE:NAME=DEFAULT_VALUE1|DEFAULT_VALUE2|...,VALUE1|VALUE2|...}
// Type is optional. Type may contain an optional argument with syntax: TYPE(ARG)
// NAME and VALUEs may contain an optional display name with syntax: NAME(DISPLAY_NAME)
// DEFAULT_VALUEs may not contain display name
// Examples: ${age} input form without default value
// ${age=3} input form with default value
// ${age(Age)=3} input form with display name and default value
// ${country=US(United States)|UK|JP} select form with
// ${checkbox( or ):country(Country)=US|JP,US(United States)|UK|JP}
// checkbox form with " or " as delimiter: will be
// expanded to "US or JP"
private static final Pattern VAR_PTN = Pattern.compile("([_])?[$][{]([^=}]*([=][^}]*)?)[}]");
private static String[] getNameAndDisplayName(String str) {
Pattern p = Pattern.compile("([^(]*)\\s*[(]([^)]*)[)]");
@ -156,17 +194,103 @@ public class Input implements Serializable {
}
private static String[] getType(String str) {
Pattern p = Pattern.compile("([^:]*)\\s*:\\s*(.*)");
Pattern p = Pattern.compile("([^:()]*)\\s*([(][^()]*[)])?\\s*:(.*)");
Matcher m = p.matcher(str.trim());
if (m == null || m.find() == false) {
return null;
}
String[] ret = new String[2];
String[] ret = new String[3];
ret[0] = m.group(1).trim();
ret[1] = m.group(2).trim();
if (m.group(2) != null) {
ret[1] = m.group(2).trim().replaceAll("[()]", "");
}
ret[2] = m.group(3).trim();
return ret;
}
private static Input getInputForm(Matcher match) {
String hiddenPart = match.group(1);
boolean hidden = false;
if ("_".equals(hiddenPart)) {
hidden = true;
}
String m = match.group(2);
String namePart;
String valuePart;
int p = m.indexOf('=');
if (p > 0) {
namePart = m.substring(0, p);
valuePart = m.substring(p + 1);
} else {
namePart = m;
valuePart = null;
}
String varName;
String displayName = null;
String type = null;
String arg = null;
Object defaultValue = "";
ParamOption[] paramOptions = null;
// get var name type
String varNamePart;
String[] typeArray = getType(namePart);
if (typeArray != null) {
type = typeArray[0];
arg = typeArray[1];
varNamePart = typeArray[2];
} else {
varNamePart = namePart;
}
// get var name and displayname
String[] varNameArray = getNameAndDisplayName(varNamePart);
if (varNameArray != null) {
varName = varNameArray[0];
displayName = varNameArray[1];
} else {
varName = varNamePart.trim();
}
// get defaultValue
if (valuePart != null) {
// find default value
int optionP = valuePart.indexOf(",");
if (optionP >= 0) { // option available
defaultValue = valuePart.substring(0, optionP);
if (type != null && type.equals("checkbox")) {
// checkbox may contain multiple default checks
defaultValue = Input.splitPipe((String) defaultValue);
}
String optionPart = valuePart.substring(optionP + 1);
String[] options = Input.splitPipe(optionPart);
paramOptions = new ParamOption[options.length];
for (int i = 0; i < options.length; i++) {
String[] optNameArray = getNameAndDisplayName(options[i]);
if (optNameArray != null) {
paramOptions[i] = new ParamOption(optNameArray[0], optNameArray[1]);
} else {
paramOptions[i] = new ParamOption(options[i], null);
}
}
} else { // no option
defaultValue = valuePart;
}
}
return new Input(varName, displayName, type, arg, defaultValue, paramOptions, hidden);
}
public static Map<String, Input> extractSimpleQueryParam(String script) {
Map<String, Input> params = new HashMap<String, Input>();
if (script == null) {
@ -174,122 +298,57 @@ public class Input implements Serializable {
}
String replaced = script;
Pattern pattern = Pattern.compile("([_])?[$][{]([^=}]*([=][^}]*)?)[}]");
Matcher match = pattern.matcher(replaced);
Matcher match = VAR_PTN.matcher(replaced);
while (match.find()) {
String hiddenPart = match.group(1);
boolean hidden = false;
if ("_".equals(hiddenPart)) {
hidden = true;
}
String m = match.group(2);
String namePart;
String valuePart;
int p = m.indexOf('=');
if (p > 0) {
namePart = m.substring(0, p);
valuePart = m.substring(p + 1);
} else {
namePart = m;
valuePart = null;
}
String varName;
String displayName = null;
String type = null;
String defaultValue = "";
ParamOption[] paramOptions = null;
// get var name type
String varNamePart;
String[] typeArray = getType(namePart);
if (typeArray != null) {
type = typeArray[0];
varNamePart = typeArray[1];
} else {
varNamePart = namePart;
}
// get var name and displayname
String[] varNameArray = getNameAndDisplayName(varNamePart);
if (varNameArray != null) {
varName = varNameArray[0];
displayName = varNameArray[1];
} else {
varName = varNamePart.trim();
}
// get defaultValue
if (valuePart != null) {
// find default value
int optionP = valuePart.indexOf(",");
if (optionP > 0) { // option available
defaultValue = valuePart.substring(0, optionP);
String optionPart = valuePart.substring(optionP + 1);
String[] options = Input.splitPipe(optionPart);
paramOptions = new ParamOption[options.length];
for (int i = 0; i < options.length; i++) {
String[] optNameArray = getNameAndDisplayName(options[i]);
if (optNameArray != null) {
paramOptions[i] = new ParamOption(optNameArray[0], optNameArray[1]);
} else {
paramOptions[i] = new ParamOption(options[i], null);
}
}
} else { // no option
defaultValue = valuePart;
}
}
Input param = new Input(varName, displayName, type, defaultValue, paramOptions, hidden);
params.put(varName, param);
Input param = getInputForm(match);
params.put(param.name, param);
}
params.remove("pql");
return params;
}
private static final String DEFAULT_DELIMITER = ",";
public static String getSimpleQuery(Map<String, Object> params, String script) {
String replaced = script;
for (String key : params.keySet()) {
Object value = params.get(key);
replaced =
replaced.replaceAll("[_]?[$][{]([^:]*[:])?" + key + "([(][^)]*[)])?(=[^}]*)?[}]",
value.toString());
}
Pattern pattern = Pattern.compile("[$][{]([^=}]*[=][^}]*)[}]");
while (true) {
Matcher match = pattern.matcher(replaced);
if (match != null && match.find()) {
String m = match.group(1);
int p = m.indexOf('=');
String replacement = m.substring(p + 1);
int optionP = replacement.indexOf(",");
if (optionP > 0) {
replacement = replacement.substring(0, optionP);
}
replaced =
replaced.replaceFirst("[_]?[$][{]"
+ m.replaceAll("[(]", ".").replaceAll("[)]", ".").replaceAll("[|]", ".") + "[}]",
replacement);
Matcher match = VAR_PTN.matcher(replaced);
while (match.find()) {
Input input = getInputForm(match);
Object value;
if (params.containsKey(input.name)) {
value = params.get(input.name);
} else {
break;
value = input.defaultValue;
}
String expanded;
if (value instanceof Object[] || value instanceof Collection) { // multi-selection
String delimiter = input.argument;
if (delimiter == null) {
delimiter = DEFAULT_DELIMITER;
}
Collection<Object> checked = value instanceof Collection ? (Collection<Object>) value
: Arrays.asList((Object[]) value);
List<Object> validChecked = new LinkedList<Object>();
for (Object o : checked) { // filter out obsolete checked values
for (ParamOption option : input.getOptions()) {
if (option.getValue().equals(o)) {
validChecked.add(o);
break;
}
}
}
params.put(input.name, validChecked);
expanded = StringUtils.join(validChecked, delimiter);
} else { // single-selection
expanded = value.toString();
}
replaced = match.replaceFirst(expanded);
match = VAR_PTN.matcher(replaced);
}
replaced = replaced.replace("[_]?[$][{]([^=}]*)[}]", "");
return replaced;
}

33
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory;
* If you want to implement new Zeppelin interpreter, extend this class
*
* Please see,
* http://zeppelin.incubator.apache.org/docs/development/writingzeppelininterpreter.html
* https://zeppelin.incubator.apache.org/docs/latest/development/writingzeppelininterpreter.html
*
* open(), close(), interpreter() is three the most important method you need to implement.
* cancel(), getProgress(), completion() is good to have
@ -121,10 +121,6 @@ public abstract class Interpreter {
* Called when interpreter is no longer used.
*/
public void destroy() {
Scheduler scheduler = getScheduler();
if (scheduler != null) {
scheduler.stop();
}
}
public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
@ -193,6 +189,33 @@ public abstract class Interpreter {
this.classloaderUrls = classloaderUrls;
}
public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
synchronized (interpreterGroup) {
for (List<Interpreter> interpreters : interpreterGroup.values()) {
boolean belongsToSameNoteGroup = false;
Interpreter interpreterFound = null;
for (Interpreter intp : interpreters) {
if (intp.getClassName().equals(className)) {
interpreterFound = intp;
}
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
if (this == p) {
belongsToSameNoteGroup = true;
}
}
if (belongsToSameNoteGroup) {
return interpreterFound;
}
}
}
return null;
}
/**
* Type of interpreter.

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -21,6 +21,7 @@ import java.util.List;
import java.util.Map;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.resource.ResourcePool;
@ -48,6 +49,7 @@ public class InterpreterContext {
private final String paragraphTitle;
private final String paragraphId;
private final String paragraphText;
private AuthenticationInfo authenticationInfo;
private final Map<String, Object> config;
private GUI gui;
private AngularObjectRegistry angularObjectRegistry;
@ -58,6 +60,7 @@ public class InterpreterContext {
String paragraphId,
String paragraphTitle,
String paragraphText,
AuthenticationInfo authenticationInfo,
Map<String, Object> config,
GUI gui,
AngularObjectRegistry angularObjectRegistry,
@ -69,6 +72,7 @@ public class InterpreterContext {
this.paragraphId = paragraphId;
this.paragraphTitle = paragraphTitle;
this.paragraphText = paragraphText;
this.authenticationInfo = authenticationInfo;
this.config = config;
this.gui = gui;
this.angularObjectRegistry = angularObjectRegistry;
@ -94,6 +98,10 @@ public class InterpreterContext {
return paragraphTitle;
}
public AuthenticationInfo getAuthenticationInfo() {
return authenticationInfo;
}
public Map<String, Object> getConfig() {
return config;
}

127
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -23,13 +23,24 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
/**
* InterpreterGroup is list of interpreters in the same group.
* InterpreterGroup is list of interpreters in the same interpreter group.
* For example spark, pyspark, sql interpreters are in the same 'spark' group
* and InterpreterGroup will have reference to these all interpreters.
*
* Remember, list of interpreters are dedicated to a note.
* (when InterpreterOption.perNoteSession==true)
* So InterpreterGroup internally manages map of [noteId, list of interpreters]
*
* A InterpreterGroup runs on interpreter process.
* And unit of interpreter instantiate, restart, bind, unbind.
*/
public class InterpreterGroup extends LinkedList<Interpreter>{
public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
String id;
Logger LOGGER = Logger.getLogger(InterpreterGroup.class);
@ -38,10 +49,14 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
ResourcePool resourcePool;
// map [notebook session, Interpreters in the group], to support per note session interpreters
//Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
// List<Interpreter>>();
private static final Map<String, InterpreterGroup> allInterpreterGroups =
new ConcurrentHashMap<String, InterpreterGroup>();
public static InterpreterGroup get(String id) {
public static InterpreterGroup getByInterpreterGroupId(String id) {
return allInterpreterGroups.get(id);
}
@ -49,11 +64,18 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
return new LinkedList(allInterpreterGroups.values());
}
/**
* Create InterpreterGroup with given id
* @param id
*/
public InterpreterGroup(String id) {
this.id = id;
allInterpreterGroups.put(id, this);
}
/**
* Create InterpreterGroup with autogenerated id
*/
public InterpreterGroup() {
getId();
allInterpreterGroups.put(id, this);
@ -73,10 +95,22 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
}
}
/**
* Get combined property of all interpreters in this group
* @return
*/
public Properties getProperty() {
Properties p = new Properties();
for (Interpreter intp : this) {
p.putAll(intp.getProperty());
Collection<List<Interpreter>> intpGroupForANote = this.values();
if (intpGroupForANote != null && intpGroupForANote.size() > 0) {
for (List<Interpreter> intpGroup : intpGroupForANote) {
for (Interpreter intp : intpGroup) {
p.putAll(intp.getProperty());
}
// it's okay to break here while every List<Interpreters> will have the same property set
break;
}
}
return p;
}
@ -97,13 +131,45 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
this.remoteInterpreterProcess = remoteInterpreterProcess;
}
/**
* Close all interpreter instances in this group
*/
public void close() {
LOGGER.info("Close interpreter group " + getId());
List<Interpreter> intpToClose = new LinkedList<Interpreter>();
for (List<Interpreter> intpGroupForNote : this.values()) {
intpToClose.addAll(intpGroupForNote);
}
close(intpToClose);
}
/**
* Close all interpreter instances in this group for the note
* @param noteId
*/
public void close(String noteId) {
LOGGER.info("Close interpreter group " + getId() + " for note " + noteId);
List<Interpreter> intpForNote = this.get(noteId);
close(intpForNote);
}
private void close(Collection<Interpreter> intpToClose) {
if (intpToClose == null) {
return;
}
List<Thread> closeThreads = new LinkedList<Thread>();
for (final Interpreter intp : this) {
for (final Interpreter intp : intpToClose) {
Thread t = new Thread() {
public void run() {
Scheduler scheduler = intp.getScheduler();
intp.close();
if (scheduler != null) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
}
};
@ -120,10 +186,46 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
}
}
/**
* Destroy all interpreter instances in this group for the note
* @param noteId
*/
public void destroy(String noteId) {
LOGGER.info("Destroy interpreter group " + getId() + " for note " + noteId);
List<Interpreter> intpForNote = this.get(noteId);
destroy(intpForNote);
}
/**
* Destroy all interpreter instances in this group
*/
public void destroy() {
LOGGER.info("Destroy interpreter group " + getId());
List<Interpreter> intpToDestroy = new LinkedList<Interpreter>();
for (List<Interpreter> intpGroupForNote : this.values()) {
intpToDestroy.addAll(intpGroupForNote);
}
destroy(intpToDestroy);
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
}
allInterpreterGroups.remove(id);
}
private void destroy(Collection<Interpreter> intpToDestroy) {
if (intpToDestroy == null) {
return;
}
List<Thread> destroyThreads = new LinkedList<Thread>();
for (final Interpreter intp : this) {
for (final Interpreter intp : intpToDestroy) {
Thread t = new Thread() {
public void run() {
intp.destroy();
@ -141,17 +243,10 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
LOGGER.error("Can't close interpreter", e);
}
}
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
}
allInterpreterGroups.remove(id);
}
public void setResourcePool(ResourcePool resourcePool) {
this.resourcePool = resourcePool;
}

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
View file

@ -19,20 +19,20 @@ package org.apache.zeppelin.interpreter.remote;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
/**
* Proxy for AngularObject that exists in remote interpreter process
*/
public class RemoteAngularObject extends AngularObject {
private transient RemoteInterpreterProcess remoteInterpreterProcess;
private transient InterpreterGroup interpreterGroup;
RemoteAngularObject(String name, Object o, String noteId, String paragraphId, String
interpreterGroupId,
AngularObjectListener listener,
RemoteInterpreterProcess remoteInterpreterProcess) {
RemoteAngularObject(String name, Object o, String noteId, String paragraphId,
InterpreterGroup interpreterGroup,
AngularObjectListener listener) {
super(name, o, noteId, paragraphId, listener);
this.remoteInterpreterProcess = remoteInterpreterProcess;
this.interpreterGroup = interpreterGroup;
}
@Override
@ -45,8 +45,9 @@ public class RemoteAngularObject extends AngularObject {
if (emitRemoteProcess) {
// send updated value to remote interpreter
remoteInterpreterProcess.updateRemoteAngularObject(getName(), getNoteId(), getParagraphId()
, o);
interpreterGroup.getRemoteInterpreterProcess().
updateRemoteAngularObject(
getName(), getNoteId(), getParagraphId(), o);
}
}
}

23
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
View file

@ -47,19 +47,7 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
}
private RemoteInterpreterProcess getRemoteInterpreterProcess() {
if (interpreterGroup.size() == 0) {
throw new RuntimeException("Can't get remoteInterpreterProcess");
}
Interpreter p = interpreterGroup.get(0);
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
if (p instanceof RemoteInterpreter) {
return ((RemoteInterpreter) p).getInterpreterProcess();
} else {
throw new RuntimeException("Can't get remoteInterpreterProcess");
}
return interpreterGroup.getRemoteInterpreterProcess();
}
/**
@ -141,12 +129,7 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
@Override
protected AngularObject createNewAngularObject(String name, Object o, String noteId, String
paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
throw new RuntimeException("Remote Interpreter process not found");
}
return new RemoteAngularObject(name, o, noteId, paragraphId, getInterpreterGroupId(),
getAngularObjectListener(),
getRemoteInterpreterProcess());
return new RemoteAngularObject(name, o, noteId, paragraphId, interpreterGroup,
getAngularObjectListener());
}
}

105
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -17,19 +17,11 @@
package org.apache.zeppelin.interpreter.remote;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import org.apache.thrift.TException;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
@ -43,7 +35,7 @@ import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
/**
*
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
@ -53,13 +45,16 @@ public class RemoteInterpreter extends Interpreter {
private String interpreterPath;
private String localRepoPath;
private String className;
private String noteId;
FormType formType;
boolean initialized;
private Map<String, String> env;
private int connectTimeout;
private int maxPoolSize;
private static String schedulerName;
public RemoteInterpreter(Properties property,
String noteId,
String className,
String interpreterRunner,
String interpreterPath,
@ -68,6 +63,7 @@ public class RemoteInterpreter extends Interpreter {
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.noteId = noteId;
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
@ -80,6 +76,7 @@ public class RemoteInterpreter extends Interpreter {
}
public RemoteInterpreter(Properties property,
String noteId,
String className,
String interpreterRunner,
String interpreterPath,
@ -89,6 +86,7 @@ public class RemoteInterpreter extends Interpreter {
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
this.noteId = noteId;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
@ -123,39 +121,36 @@ public class RemoteInterpreter extends Interpreter {
}
}
private synchronized void init() {
public synchronized void init() {
if (initialized == true) {
return;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
int rc = interpreterProcess.reference(getInterpreterGroup());
interpreterProcess.setMaxPoolSize(this.maxPoolSize);
synchronized (interpreterProcess) {
// when first process created
if (rc == 1) {
// create all interpreter class in this interpreter group
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
for (Interpreter intp : this.getInterpreterGroup()) {
logger.info("Create remote interpreter {}", intp.getClassName());
property.put("zeppelin.interpreter.localRepo", localRepoPath);
client.createInterpreter(getInterpreterGroup().getId(),
intp.getClassName(), (Map) property);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
interpreterProcess.reference(getInterpreterGroup());
interpreterProcess.setMaxPoolSize(
Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
synchronized (interpreterProcess) {
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
logger.info("Create remote interpreter {}", getClassName());
property.put("zeppelin.interpreter.localRepo", localRepoPath);
client.createInterpreter(getInterpreterGroup().getId(), noteId,
getClassName(), (Map) property);
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
initialized = true;
@ -165,19 +160,31 @@ public class RemoteInterpreter extends Interpreter {
@Override
public void open() {
init();
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (interpreterGroup) {
// initialize all interpreters in this interpreter group
List<Interpreter> interpreters = interpreterGroup.get(noteId);
for (Interpreter intp : interpreters) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
((RemoteInterpreter) p).init();
}
}
}
@Override
public void close() {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
if (client != null) {
client.close(className);
client.close(noteId, className);
}
} catch (TException e) {
broken = true;
@ -219,7 +226,8 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
GUI settings = context.getGui();
RemoteInterpreterResult remoteResult = client.interpret(className, st, convert(context));
RemoteInterpreterResult remoteResult = client.interpret(
noteId, className, st, convert(context));
Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
@ -256,7 +264,7 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
client.cancel(className, convert(context));
client.cancel(noteId, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
@ -284,7 +292,7 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
formType = FormType.valueOf(client.getFormType(className));
formType = FormType.valueOf(client.getFormType(noteId, className));
return formType;
} catch (TException e) {
broken = true;
@ -310,7 +318,7 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
return client.getProgress(className, convert(context));
return client.getProgress(noteId, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
@ -332,7 +340,7 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
return client.completion(className, buf, cursor);
return client.completion(noteId, className, buf, cursor);
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
@ -349,7 +357,9 @@ public class RemoteInterpreter extends Interpreter {
return null;
} else {
return SchedulerFactory.singleton().createOrGetRemoteScheduler(
"remoteinterpreter_" + interpreterProcess.hashCode(), interpreterProcess,
RemoteInterpreter.class.getName() + noteId + interpreterProcess.hashCode(),
noteId,
interpreterProcess,
maxConcurrency);
}
}
@ -364,6 +374,7 @@ public class RemoteInterpreter extends Interpreter {
ic.getParagraphId(),
ic.getParagraphTitle(),
ic.getParagraphText(),
gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()),
gson.toJson(ic.getGui()),
gson.toJson(ic.getRunners()));

3
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -244,7 +244,8 @@ public class RemoteInterpreterEventPoller extends Thread {
}
private Object getResource(ResourceId resourceId) {
InterpreterGroup intpGroup = InterpreterGroup.get(resourceId.getResourcePoolId());
InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}

9
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -281,6 +281,15 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
clientPool.setMaxTotal(size + 2);
}
}
public int getMaxPoolSize() {
if (clientPool != null) {
return clientPool.getMaxTotal();
} else {
return 0;
}
}
/**
* Called when angular object is updated in client side to propagate
* change to the remote process

95
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -23,20 +23,13 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import org.apache.thrift.TException;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.*;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
@ -49,6 +42,7 @@ import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.scheduler.JobProgressPoller;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -142,10 +136,9 @@ public class RemoteInterpreterServer
@Override
public void createInterpreter(String interpreterGroupId, String className, Map<String, String>
properties)
throws TException {
public void createInterpreter(String interpreterGroupId, String noteId, String
className,
Map<String, String> properties) throws TException {
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this);
@ -167,8 +160,13 @@ public class RemoteInterpreterServer
repl.setClassloaderUrls(new URL[]{});
synchronized (interpreterGroup) {
interpreterGroup.add(new LazyOpenInterpreter(
new ClassloaderInterpreter(repl, cl)));
List<Interpreter> interpreters = interpreterGroup.get(noteId);
if (interpreters == null) {
interpreters = new LinkedList<Interpreter>();
interpreterGroup.put(noteId, interpreters);
}
interpreters.add(new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl)));
}
logger.info("Instantiate interpreter {}", className);
@ -181,9 +179,18 @@ public class RemoteInterpreterServer
}
}
private Interpreter getInterpreter(String className) throws TException {
private Interpreter getInterpreter(String noteId, String className) throws TException {
if (interpreterGroup == null) {
throw new TException(
new InterpreterException("Interpreter instance " + className + " not created"));
}
synchronized (interpreterGroup) {
for (Interpreter inp : interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(noteId);
if (interpreters == null) {
throw new TException(
new InterpreterException("Interpreter " + className + " not initialized"));
}
for (Interpreter inp : interpreters) {
if (inp.getClassName().equals(className)) {
return inp;
}
@ -194,23 +201,35 @@ public class RemoteInterpreterServer
}
@Override
public void open(String className) throws TException {
Interpreter intp = getInterpreter(className);
public void open(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
intp.open();
}
@Override
public void close(String className) throws TException {
Interpreter intp = getInterpreter(className);
intp.close();
public void close(String noteId, String className) throws TException {
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(noteId);
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
while (it.hasNext()) {
Interpreter inp = it.next();
if (inp.getClassName().equals(className)) {
inp.close();
it.remove();
break;
}
}
}
}
}
@Override
public RemoteInterpreterResult interpret(String className, String st,
public RemoteInterpreterResult interpret(String noteId, String className, String st,
RemoteInterpreterContext interpreterContext) throws TException {
logger.debug("st: {}", st);
Interpreter intp = getInterpreter(className);
Interpreter intp = getInterpreter(noteId, className);
InterpreterContext context = convert(interpreterContext);
Scheduler scheduler = intp.getScheduler();
@ -343,10 +362,10 @@ public class RemoteInterpreterServer
@Override
public void cancel(String className, RemoteInterpreterContext interpreterContext)
public void cancel(String noteId, String className, RemoteInterpreterContext interpreterContext)
throws TException {
logger.info("cancel {} {}", className, interpreterContext.getParagraphId());
Interpreter intp = getInterpreter(className);
Interpreter intp = getInterpreter(noteId, className);
String jobId = interpreterContext.getParagraphId();
Job job = intp.getScheduler().removeFromWaitingQueue(jobId);
@ -358,22 +377,24 @@ public class RemoteInterpreterServer
}
@Override
public int getProgress(String className, RemoteInterpreterContext interpreterContext)
public int getProgress(String noteId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
Interpreter intp = getInterpreter(className);
Interpreter intp = getInterpreter(noteId, className);
return intp.getProgress(convert(interpreterContext));
}
@Override
public String getFormType(String className) throws TException {
Interpreter intp = getInterpreter(className);
public String getFormType(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
return intp.getFormType().toString();
}
@Override
public List<String> completion(String className, String buf, int cursor) throws TException {
Interpreter intp = getInterpreter(className);
public List<String> completion(String noteId, String className, String buf, int cursor)
throws TException {
Interpreter intp = getInterpreter(noteId, className);
return intp.completion(buf, cursor);
}
@ -392,6 +413,7 @@ public class RemoteInterpreterServer
ric.getParagraphId(),
ric.getParagraphTitle(),
ric.getParagraphText(),
gson.fromJson(ric.getAuthenticationInfo(), AuthenticationInfo.class),
(Map<String, Object>) gson.fromJson(ric.getConfig(),
new TypeToken<Map<String, Object>>() {}.getType()),
gson.fromJson(ric.getGui(), GUI.class),
@ -442,14 +464,19 @@ public class RemoteInterpreterServer
}
@Override
public String getStatus(String jobId)
public String getStatus(String noteId, String jobId)
throws TException {
if (interpreterGroup == null) {
return "Unknown";
}
synchronized (interpreterGroup) {
for (Interpreter intp : interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(noteId);
if (interpreters == null) {
return "Unknown";
}
for (Interpreter intp : interpreters) {
for (Job job : intp.getScheduler().getJobsRunning()) {
if (jobId.equals(job.getId())) {
return job.getStatus().name();

147
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java
View file

@ -16,7 +16,7 @@
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
* Autogenerated by Thrift Compiler (0.9.3)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-24")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-02-16")
public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteInterpreterContext, RemoteInterpreterContext._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterContext> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext");
@ -59,9 +59,10 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TITLE_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphTitle", org.apache.thrift.protocol.TType.STRING, (short)3);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphText", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField CONFIG_FIELD_DESC = new org.apache.thrift.protocol.TField("config", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField GUI_FIELD_DESC = new org.apache.thrift.protocol.TField("gui", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.protocol.TField RUNNERS_FIELD_DESC = new org.apache.thrift.protocol.TField("runners", org.apache.thrift.protocol.TType.STRING, (short)7);
private static final org.apache.thrift.protocol.TField AUTHENTICATION_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("authenticationInfo", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField CONFIG_FIELD_DESC = new org.apache.thrift.protocol.TField("config", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.protocol.TField GUI_FIELD_DESC = new org.apache.thrift.protocol.TField("gui", org.apache.thrift.protocol.TType.STRING, (short)7);
private static final org.apache.thrift.protocol.TField RUNNERS_FIELD_DESC = new org.apache.thrift.protocol.TField("runners", org.apache.thrift.protocol.TType.STRING, (short)8);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@ -73,6 +74,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
public String paragraphId; // required
public String paragraphTitle; // required
public String paragraphText; // required
public String authenticationInfo; // required
public String config; // required
public String gui; // required
public String runners; // required
@ -83,9 +85,10 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
PARAGRAPH_ID((short)2, "paragraphId"),
PARAGRAPH_TITLE((short)3, "paragraphTitle"),
PARAGRAPH_TEXT((short)4, "paragraphText"),
CONFIG((short)5, "config"),
GUI((short)6, "gui"),
RUNNERS((short)7, "runners");
AUTHENTICATION_INFO((short)5, "authenticationInfo"),
CONFIG((short)6, "config"),
GUI((short)7, "gui"),
RUNNERS((short)8, "runners");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@ -108,11 +111,13 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return PARAGRAPH_TITLE;
case 4: // PARAGRAPH_TEXT
return PARAGRAPH_TEXT;
case 5: // CONFIG
case 5: // AUTHENTICATION_INFO
return AUTHENTICATION_INFO;
case 6: // CONFIG
return CONFIG;
case 6: // GUI
case 7: // GUI
return GUI;
case 7: // RUNNERS
case 8: // RUNNERS
return RUNNERS;
default:
return null;
@ -165,6 +170,8 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.PARAGRAPH_TEXT, new org.apache.thrift.meta_data.FieldMetaData("paragraphText", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.AUTHENTICATION_INFO, new org.apache.thrift.meta_data.FieldMetaData("authenticationInfo", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.CONFIG, new org.apache.thrift.meta_data.FieldMetaData("config", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.GUI, new org.apache.thrift.meta_data.FieldMetaData("gui", org.apache.thrift.TFieldRequirementType.DEFAULT,
@ -183,6 +190,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
String paragraphId,
String paragraphTitle,
String paragraphText,
String authenticationInfo,
String config,
String gui,
String runners)
@ -192,6 +200,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
this.paragraphId = paragraphId;
this.paragraphTitle = paragraphTitle;
this.paragraphText = paragraphText;
this.authenticationInfo = authenticationInfo;
this.config = config;
this.gui = gui;
this.runners = runners;
@ -213,6 +222,9 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (other.isSetParagraphText()) {
this.paragraphText = other.paragraphText;
}
if (other.isSetAuthenticationInfo()) {
this.authenticationInfo = other.authenticationInfo;
}
if (other.isSetConfig()) {
this.config = other.config;
}
@ -234,6 +246,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
this.paragraphId = null;
this.paragraphTitle = null;
this.paragraphText = null;
this.authenticationInfo = null;
this.config = null;
this.gui = null;
this.runners = null;
@ -335,6 +348,30 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
}
public String getAuthenticationInfo() {
return this.authenticationInfo;
}
public RemoteInterpreterContext setAuthenticationInfo(String authenticationInfo) {
this.authenticationInfo = authenticationInfo;
return this;
}
public void unsetAuthenticationInfo() {
this.authenticationInfo = null;
}
/** Returns true if field authenticationInfo is set (has been assigned a value) and false otherwise */
public boolean isSetAuthenticationInfo() {
return this.authenticationInfo != null;
}
public void setAuthenticationInfoIsSet(boolean value) {
if (!value) {
this.authenticationInfo = null;
}
}
public String getConfig() {
return this.config;
}
@ -441,6 +478,14 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
break;
case AUTHENTICATION_INFO:
if (value == null) {
unsetAuthenticationInfo();
} else {
setAuthenticationInfo((String)value);
}
break;
case CONFIG:
if (value == null) {
unsetConfig();
@ -482,6 +527,9 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
case PARAGRAPH_TEXT:
return getParagraphText();
case AUTHENTICATION_INFO:
return getAuthenticationInfo();
case CONFIG:
return getConfig();
@ -510,6 +558,8 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return isSetParagraphTitle();
case PARAGRAPH_TEXT:
return isSetParagraphText();
case AUTHENTICATION_INFO:
return isSetAuthenticationInfo();
case CONFIG:
return isSetConfig();
case GUI:
@ -569,6 +619,15 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return false;
}
boolean this_present_authenticationInfo = true && this.isSetAuthenticationInfo();
boolean that_present_authenticationInfo = true && that.isSetAuthenticationInfo();
if (this_present_authenticationInfo || that_present_authenticationInfo) {
if (!(this_present_authenticationInfo && that_present_authenticationInfo))
return false;
if (!this.authenticationInfo.equals(that.authenticationInfo))
return false;
}
boolean this_present_config = true && this.isSetConfig();
boolean that_present_config = true && that.isSetConfig();
if (this_present_config || that_present_config) {
@ -623,6 +682,11 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (present_paragraphText)
list.add(paragraphText);
boolean present_authenticationInfo = true && (isSetAuthenticationInfo());
list.add(present_authenticationInfo);
if (present_authenticationInfo)
list.add(authenticationInfo);
boolean present_config = true && (isSetConfig());
list.add(present_config);
if (present_config)
@ -689,6 +753,16 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetAuthenticationInfo()).compareTo(other.isSetAuthenticationInfo());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetAuthenticationInfo()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.authenticationInfo, other.authenticationInfo);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetConfig()).compareTo(other.isSetConfig());
if (lastComparison != 0) {
return lastComparison;
@ -771,6 +845,14 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
first = false;
if (!first) sb.append(", ");
sb.append("authenticationInfo:");
if (this.authenticationInfo == null) {
sb.append("null");
} else {
sb.append(this.authenticationInfo);
}
first = false;
if (!first) sb.append(", ");
sb.append("config:");
if (this.config == null) {
sb.append("null");
@ -869,7 +951,15 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 5: // CONFIG
case 5: // AUTHENTICATION_INFO
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.authenticationInfo = iprot.readString();
struct.setAuthenticationInfoIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 6: // CONFIG
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.config = iprot.readString();
struct.setConfigIsSet(true);
@ -877,7 +967,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 6: // GUI
case 7: // GUI
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.gui = iprot.readString();
struct.setGuiIsSet(true);
@ -885,7 +975,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 7: // RUNNERS
case 8: // RUNNERS
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.runners = iprot.readString();
struct.setRunnersIsSet(true);
@ -928,6 +1018,11 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
oprot.writeString(struct.paragraphText);
oprot.writeFieldEnd();
}
if (struct.authenticationInfo != null) {
oprot.writeFieldBegin(AUTHENTICATION_INFO_FIELD_DESC);
oprot.writeString(struct.authenticationInfo);
oprot.writeFieldEnd();
}
if (struct.config != null) {
oprot.writeFieldBegin(CONFIG_FIELD_DESC);
oprot.writeString(struct.config);
@ -973,16 +1068,19 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (struct.isSetParagraphText()) {
optionals.set(3);
}
if (struct.isSetConfig()) {
if (struct.isSetAuthenticationInfo()) {
optionals.set(4);
}
if (struct.isSetGui()) {
if (struct.isSetConfig()) {
optionals.set(5);
}
if (struct.isSetRunners()) {
if (struct.isSetGui()) {
optionals.set(6);
}
oprot.writeBitSet(optionals, 7);
if (struct.isSetRunners()) {
optionals.set(7);
}
oprot.writeBitSet(optionals, 8);
if (struct.isSetNoteId()) {
oprot.writeString(struct.noteId);
}
@ -995,6 +1093,9 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (struct.isSetParagraphText()) {
oprot.writeString(struct.paragraphText);
}
if (struct.isSetAuthenticationInfo()) {
oprot.writeString(struct.authenticationInfo);
}
if (struct.isSetConfig()) {
oprot.writeString(struct.config);
}
@ -1009,7 +1110,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterContext struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(7);
BitSet incoming = iprot.readBitSet(8);
if (incoming.get(0)) {
struct.noteId = iprot.readString();
struct.setNoteIdIsSet(true);
@ -1027,14 +1128,18 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
struct.setParagraphTextIsSet(true);
}
if (incoming.get(4)) {
struct.authenticationInfo = iprot.readString();
struct.setAuthenticationInfoIsSet(true);
}
if (incoming.get(5)) {
struct.config = iprot.readString();
struct.setConfigIsSet(true);
}
if (incoming.get(5)) {
if (incoming.get(6)) {
struct.gui = iprot.readString();
struct.setGuiIsSet(true);
}
if (incoming.get(6)) {
if (incoming.get(7)) {
struct.runners = iprot.readString();
struct.setRunnersIsSet(true);
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java
View file

@ -16,7 +16,7 @@
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
* Autogenerated by Thrift Compiler (0.9.3)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-24")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-02-16")
public class RemoteInterpreterEvent implements org.apache.thrift.TBase<RemoteInterpreterEvent, RemoteInterpreterEvent._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterEvent> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent");

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java
View file

@ -16,7 +16,7 @@
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
* Autogenerated by Thrift Compiler (0.9.3)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java
View file

@ -16,7 +16,7 @@
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
* Autogenerated by Thrift Compiler (0.9.3)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-24")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-02-16")
public class RemoteInterpreterResult implements org.apache.thrift.TBase<RemoteInterpreterResult, RemoteInterpreterResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResult> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult");

1379
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java
View file

File diff suppressed because it is too large Load diff

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
View file

@ -45,14 +45,16 @@ public class RemoteScheduler implements Scheduler {
boolean terminate = false;
private String name;
private int maxConcurrency;
private final String noteId;
private RemoteInterpreterProcess interpreterProcess;
public RemoteScheduler(String name, ExecutorService executor,
public RemoteScheduler(String name, ExecutorService executor, String noteId,
RemoteInterpreterProcess interpreterProcess, SchedulerListener listener,
int maxConcurrency) {
this.name = name;
this.executor = executor;
this.listener = listener;
this.noteId = noteId;
this.interpreterProcess = interpreterProcess;
this.maxConcurrency = maxConcurrency;
}
@ -257,7 +259,7 @@ public class RemoteScheduler implements Scheduler {
boolean broken = false;
try {
String statusStr = client.getStatus(job.getId());
String statusStr = client.getStatus(noteId, job.getId());
if ("Unknown".equals(statusStr)) {
// not found this job in the remote schedulers.
// maybe not submitted, maybe already finished

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
View file

@ -86,6 +86,7 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetRemoteScheduler(
String name,
String noteId,
RemoteInterpreterProcess interpreterProcess,
int maxConcurrency) {
@ -94,6 +95,7 @@ public class SchedulerFactory implements SchedulerListener {
Scheduler s = new RemoteScheduler(
name,
executor,
noteId,
interpreterProcess,
this,
maxConcurrency);

55
zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java Normal file
View file

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.user;
/***
*
*/
public class AuthenticationInfo {
String user;
String ticket;
public AuthenticationInfo() {}
/***
*
* @param user
* @param ticket
*/
public AuthenticationInfo(String user, String ticket) {
this.user = user;
this.ticket = ticket;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getTicket() {
return ticket;
}
public void setTicket(String ticket) {
this.ticket = ticket;
}
}

25
zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift
View file

@ -24,9 +24,10 @@ struct RemoteInterpreterContext {
2: string paragraphId,
3: string paragraphTitle,
4: string paragraphText,
5: string config, // json serialized config
6: string gui, // json serialized gui
7: string runners // json serialized runner
5: string authenticationInfo,
6: string config, // json serialized config
7: string gui, // json serialized gui
8: string runners // json serialized runner
}
struct RemoteInterpreterResult {
@ -55,18 +56,18 @@ struct RemoteInterpreterEvent {
}
service RemoteInterpreterService {
void createInterpreter(1: string intpGroupId, 2: string className, 3: map<string, string> properties);
void createInterpreter(1: string intpGroupId, 2: string noteId, 3: string className, 4: map<string, string> properties);
void open(1: string className);
void close(1: string className);
RemoteInterpreterResult interpret(1: string className, 2: string st, 3: RemoteInterpreterContext interpreterContext);
void cancel(1: string className, 2: RemoteInterpreterContext interpreterContext);
i32 getProgress(1: string className, 2: RemoteInterpreterContext interpreterContext);
string getFormType(1: string className);
list<string> completion(1: string className, 2: string buf, 3: i32 cursor);
void open(1: string noteId, 2: string className);
void close(1: string noteId, 2: string className);
RemoteInterpreterResult interpret(1: string noteId, 2: string className, 3: string st, 4: RemoteInterpreterContext interpreterContext);
void cancel(1: string noteId, 2: string className, 3: RemoteInterpreterContext interpreterContext);
i32 getProgress(1: string noteId, 2: string className, 3: RemoteInterpreterContext interpreterContext);
string getFormType(1: string noteId, 2: string className);
list<string> completion(1: string noteId, 2: string className, 3: string buf, 4: i32 cursor);
void shutdown();
string getStatus(1:string jobId);
string getStatus(1: string noteId, 2:string jobId);
RemoteInterpreterEvent getEvent();

93
zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java
View file

@ -17,12 +17,19 @@
package org.apache.zeppelin.display;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.apache.zeppelin.display.Input.ParamOption;
public class InputTest {
@Before
@ -34,6 +41,88 @@ public class InputTest {
}
@Test
public void testDefaultParamReplace() throws IOException{
public void testFormExtraction() {
// input form
String script = "${input_form=}";
Map<String, Input> forms = Input.extractSimpleQueryParam(script);
assertEquals(1, forms.size());
Input form = forms.get("input_form");
assertEquals("input_form", form.name);
assertNull(form.displayName);
assertEquals("", form.defaultValue);
assertNull(form.options);
// input form with display name & default value
script = "${input_form(Input Form)=xxx}";
forms = Input.extractSimpleQueryParam(script);
form = forms.get("input_form");
assertEquals("xxx", form.defaultValue);
// selection form
script = "${select_form(Selection Form)=op1,op1|op2(Option 2)|op3}";
form = Input.extractSimpleQueryParam(script).get("select_form");
assertEquals("select_form", form.name);
assertEquals("op1", form.defaultValue);
assertArrayEquals(new ParamOption[]{new ParamOption("op1", null),
new ParamOption("op2", "Option 2"), new ParamOption("op3", null)}, form.options);
// checkbox form
script = "${checkbox:checkbox_form=op1,op1|op2|op3}";
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
assertEquals("checkbox_form", form.name);
assertEquals("checkbox", form.type);
assertArrayEquals(new Object[]{"op1"}, (Object[]) form.defaultValue);
assertArrayEquals(new ParamOption[]{new ParamOption("op1", null),
new ParamOption("op2", null), new ParamOption("op3", null)}, form.options);
// checkbox form with multiple default checks
script = "${checkbox:checkbox_form(Checkbox Form)=op1|op3,op1(Option 1)|op2|op3}";
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
assertEquals("checkbox_form", form.name);
assertEquals("Checkbox Form", form.displayName);
assertEquals("checkbox", form.type);
assertArrayEquals(new Object[]{"op1", "op3"}, (Object[]) form.defaultValue);
assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"),
new ParamOption("op2", null), new ParamOption("op3", null)}, form.options);
// checkbox form with no default check
script = "${checkbox:checkbox_form(Checkbox Form)=,op1(Option 1)|op2(Option 2)|op3(Option 3)}";
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
assertEquals("checkbox_form", form.name);
assertEquals("Checkbox Form", form.displayName);
assertEquals("checkbox", form.type);
assertArrayEquals(new Object[]{}, (Object[]) form.defaultValue);
assertArrayEquals(new ParamOption[]{new ParamOption("op1", "Option 1"),
new ParamOption("op2", "Option 2"), new ParamOption("op3", "Option 3")}, form.options);
}
@Test
public void testFormSubstitution() {
// test form substitution without new forms
String script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" +
"CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}";
Map<String, Object> params = new HashMap<String, Object>();
params.put("input_form", "some_input");
params.put("select_form", "s_op2");
params.put("checkbox_form", new String[]{"c_op1", "c_op3"});
String replaced = Input.getSimpleQuery(params, script);
assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3", replaced);
// test form substitution with new forms
script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" +
"CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}\n" +
"NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}";
replaced = Input.getSimpleQuery(params, script);
assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1,c_op3\n" +
"NEW_CHECKED=nc_a and nc_c", replaced);
// test form substitution with obsoleted values
script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" +
"CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3_new}\n" +
"NEW_CHECKED=${checkbox( and ):new_check=nc_a|nc_c,nc_a|nc_b|nc_c}";
replaced = Input.getSimpleQuery(params, script);
assertEquals("INPUT=some_inputSELECTED=s_op2\nCHECKED=c_op1\n" +
"NEW_CHECKED=nc_a and nc_c", replaced);
}
}

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java
View file

@ -27,7 +27,7 @@ public class InterpreterContextTest {
public void testThreadLocal() {
assertNull(InterpreterContext.get());
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null, null));
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null));
assertNotNull(InterpreterContext.get());
InterpreterContext.remove();

Some files were not shown because too many files have changed in this diff Show more