Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge with master

Browse source
This commit is contained in:
Prasad Wagle 2016-02-25 15:38:23 -08:00
commit 29ebf486fc
190 changed files with 11724 additions and 1771 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored
View file

@ -8,6 +8,13 @@
# interpreter
/interpreter/
# interpreter temp files
spark/derby.log
spark/metastore_db
spark-1.*-bin-hadoop*
lens/lens-cli-hist.log
# conf file
conf/zeppelin-env.sh
conf/zeppelin-site.xml
@ -45,6 +52,7 @@ zeppelin-web/bower_components
**/data/
**/build/
**/testing/
!/testing/
# OS generated files #
######################

21
.travis.yml
View file

@ -19,27 +19,31 @@ matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.1
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.0
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "export DISPLAY=:99.0"
@ -49,11 +53,12 @@ install:
- mvn $BUILD_FLAG $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
after_failure:
- cat target/rat.txt

10
CONTRIBUTING.md
View file

@ -7,6 +7,10 @@ Contributing to Zeppelin (Source code, Documents, Image, Website) means you agre
2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
3. Contribute your patch via Pull Request.
Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it.
For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful.
## Creating a Pull Request
In order to make the review process easier, please follow this template when making a Pull Request:
@ -144,13 +148,13 @@ First of all, you need the Zeppelin source code. The official location for Zeppe
Get the source code on your development machine using git.
```
git clone http://git.apache.org/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 http://git.apache.org/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
or with write access

2
NOTICE
View file

@ -1,5 +1,5 @@
Apache Zeppelin (incubating)
Copyright 2015 The Apache Software Foundation
Copyright 2015 - 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).

2
README.md
View file

@ -104,7 +104,7 @@ minor version can be adjusted by `-Dhadoop.version=x.x.x`
##### -Pyarn (optional)
enable YARN support for local mode
> YARN for local mode is not supported for Spark v1.5.0 or higher. Set SPARK_HOME instead.
##### -Ppyspark (optional)

9
SECURITY-README.md
View file

@ -33,11 +33,8 @@ The scope of this PR is to require credentials to access Zeppelin. To achieve th
Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info).
## Websocket security
Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach:
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above).

4
bin/common.sh
View file

@ -36,10 +36,6 @@ if [[ -z "${ZEPPELIN_LOG_DIR}" ]]; then
export ZEPPELIN_LOG_DIR="${ZEPPELIN_HOME}/logs"
fi
if [[ -z "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
export ZEPPELIN_NOTEBOOK_DIR="${ZEPPELIN_HOME}/notebook"
fi
if [[ -z "$ZEPPELIN_PID_DIR" ]]; then
export ZEPPELIN_PID_DIR="${ZEPPELIN_HOME}/run"
fi

2
bin/functions.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information

9
bin/interpreter.sh
View file

@ -21,10 +21,10 @@ bin=$(cd "${bin}">/dev/null; pwd)
function usage() {
echo "usage) $0 -p <port> -d <directory to load>"
echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
}
while getopts "hp:d:" o; do
while getopts "hp:d:l:" o; do
case ${o} in
h)
usage
@ -36,6 +36,9 @@ while getopts "hp:d:" o; do
p)
PORT=${OPTARG}
;;
l)
LOCAL_INTERPRETER_REPO=${OPTARG}
;;
esac
done
@ -128,6 +131,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
fi
fi
addJarInDir "${LOCAL_INTERPRETER_REPO}"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
if [[ -n "${SPARK_SUBMIT}" ]]; then

22
bin/zeppelin-daemon.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@ -21,7 +19,7 @@
# description: Start and stop daemon script for.
#
USAGE="Usage: zeppelin-daemon.sh [--config <conf-dir>] {start|stop|restart|reload|status}"
USAGE="Usage: zeppelin-daemon.sh [--config <conf-dir>] {start|stop|upstart|restart|reload|status}"
if [[ "$1" == "--config" ]]; then
shift
@ -93,11 +91,6 @@ function initialize_default_directories() {
echo "Pid dir doesn't exist, create ${ZEPPELIN_PID_DIR}"
$(mkdir -p "${ZEPPELIN_PID_DIR}")
fi
if [[ ! -d "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
echo "Notebook dir doesn't exist, create ${ZEPPELIN_NOTEBOOK_DIR}"
$(mkdir -p "${ZEPPELIN_NOTEBOOK_DIR}")
fi
}
function wait_for_zeppelin_to_die() {
@ -159,6 +152,16 @@ function check_if_process_is_alive() {
fi
}
function upstart() {
# upstart() allows zeppelin to be run and managed as a service
# for example, this could be called from an upstart script in /etc/init
# where the service manager starts and stops the process
initialize_default_directories
$ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_MAIN >> "${ZEPPELIN_OUTFILE}"
}
function start() {
local pid
@ -241,6 +244,9 @@ case "${1}" in
stop)
stop
;;
upstart)
upstart
;;
reload)
stop
start

2
bin/zeppelin.sh
View file

@ -1,7 +1,5 @@
#!/bin/bash
#
# Copyright 2007 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information

6
conf/zeppelin-env.sh.template
View file

@ -29,10 +29,11 @@
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false"
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
#### Spark interpreter configuration ####
@ -42,6 +43,7 @@
##
# export SPARK_HOME # (required) When it is defined, load it instead of Zeppelin embedded Spark libraries
# export SPARK_SUBMIT_OPTIONS # (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G".
# export SPARK_APP_NAME # (optional) The name of spark application.
## Use embedded spark binaries ##
## without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries.

35
conf/zeppelin-site.xml.template
View file

@ -83,6 +83,33 @@
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>
<name>zeppelin.notebook.azure.user</name>
<value>user</value>
<description>optional user name for Azure folder structure</description>
</property>
<property>
<name>zeppelin.notebook.azure.share</name>
<value>zeppelin</value>
<description>share name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.azure.connectionString</name>
<value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
<description>share name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.AzureNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
-->
<!-- For versioning your local norebook storage using Git repository
<property>
<name>zeppelin.notebook.storage</name>
@ -103,9 +130,15 @@
<description>Interpreter implementation base directory</description>
</property>
<property>
<name>zeppelin.interpreter.localRepo</name>
<value>local-repo</value>
<description>Local repository for interpreter's additional dependency loading</description>
</property>
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

39
docs/README.md
View file

@ -8,10 +8,13 @@ See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
**tl;dr version:**
```
ruby --version >= 1.9.3
gem install bundler
# go to /docs under your Zeppelin source
bundle install
```
*On OS X 10.9 you may need to do "xcode-select --install"*
@ -20,28 +23,28 @@ See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
bundle exec jekyll serve --watch
## Deploy to ASF svnpubsub infra (commiters only)
1. generate static website in `./_site`
```
bundle exec jekyll build --safe
```
2. checkout ASF repo
```
svn co https://svn.apache.org/repos/asf/incubator/zeppelin asf-zepplelin
```
3. copy zeppelin/_site to asf-zepplelin/site/docs/[VERSION]
4. ```svn commit```
## Adding a new page
rake page name="new-page.md"
## Bumping up version in a new release
## Bumping up version
* `ZEPPELIN_VERSION` and `BASE_PATH` property in _config.yml
* `Zeppelin <small>([VERSION])</small>` in _includes/themes/zeppelin/_navigation.html
should be updated
* `BASE_PATH` property in _config.yml
* `ZEPPELIN <small>([VERSION])</small>` in _includes/themes/zeppelin/_navigation.html
need to be updated
## Deploy to ASF svnpubsub infra (for committers only)
1. generate static website in `./_site`
```
# go to /docs under Zeppelin source
bundle exec jekyll build --safe
```
2. checkout ASF repo
```
svn co https://svn.apache.org/repos/asf/incubator/zeppelin asf-zeppelin
```
3. copy `zeppelin/docs/_site` to `asf-zeppelin/site/docs/[VERSION]`
4. ```svn commit```

2
docs/_config.yml
View file

@ -21,6 +21,8 @@ author :
twitter : ASF
feedburner : feedname
ZEPPELIN_VERSION : 0.6.0-incubating-SNAPSHOT
# The production_url is only used when full-domain names are needed
# such as sitemap.txt
# Most places will/should use BASE_PATH to make the urls

12
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -31,20 +31,22 @@
<li role="separator" class="divider"></li>
<!-- li><span><b>Guide</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/dynamicform.html">Dynamic Form</a></li>
<li><a href="{{BASE_PATH}}/manual/publish.html">Publish your Paragraph</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
<li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
<li><a href="{{BASE_PATH}}/interpreter/geode.html">Geode</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
@ -53,6 +55,9 @@
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
<li><a href="{{BASE_PATH}}/interpreter/tachyon.html">Tachyon</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Tajo</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
</ul>
</li>
<li>
@ -89,9 +94,12 @@
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Shiro Security</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/shiroauthentication.html">Shiro Authentication</a></li>
</ul>
</li>
</ul>
</nav><!--/.navbar-collapse -->
</div>
</div>
</div>

4
docs/_includes/themes/zeppelin/default.html
View file

@ -11,9 +11,11 @@
<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet">
<!-- Le styles -->
<link href="{{ ASSET_PATH }}/bootstrap/css/bootstrap.css" rel="stylesheet">
<link href="{{ ASSET_PATH }}/css/style.css?body=1" rel="stylesheet" type="text/css">

BIN
docs/assets/themes/zeppelin/img/docs-img/basic-usage-angular.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 94 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/copy-the-link.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 185 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 298 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 328 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 369 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/jdbc-multi-connection-setting.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 104 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/jdbc-simple-connection-setting.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 80 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 7 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/link-the-paragraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 311 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/string-converter-angular.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 133 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/your-website.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 425 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin-login.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

58
docs/development/howtocontribute.md
View file

@ -5,52 +5,49 @@ description: "How to contribute"
group: development
---
# Contributing to Apache Zeppelin ( Code )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
## Setting up
Here are some tools you will need to build and test Zeppelin.
#### Software Configuration Management ( SCM )
### Setting up
Here are some things you will need to do to build and test Zeppelin.
Since Zeppelin uses Git for it's SCM system, you need git client installed in your development machine.
#### Software Configuration Management(SCM)
Zeppelin uses Git for it's SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` You'll need git client installed in your development machine.
#### Integrated Development Environment(IDE)
#### Integrated Development Environment ( IDE )
You are free to use whatever IDE you prefer, or your favorite command line editor.
#### Build Tools
### Build Tools
To build the code, install
Oracle Java 7
Apache Maven
* Oracle Java 7
* Apache Maven
### Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
## Getting the source code
First of all, you need Zeppelin source code. The official location of Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
#### git access
### git access
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 https://github.com/apache/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
#### Fork repository
If you want not only build Zeppelin but also make changes, then you need to fork Zeppelin repository and make pull request.
If you want not only build Zeppelin but also make any changes, then you need fork [Zeppelin github mirror repository](https://github.com/apache/incubator-zeppelin) and make a pull request.
###Build
@ -67,7 +64,7 @@ mvn install -DskipTests
To build with specific spark / hadoop version
```
mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x
```
### Run Zeppelin server in development mode
@ -76,7 +73,8 @@ mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
NOTE: make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory otherwise your server build will fail to find the required dependencies in the local repro
> **Note:** Make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro.
or use daemon script
@ -84,15 +82,13 @@ or use daemon script
bin/zeppelin-daemon start
```
Server will be run on http://localhost:8080
Server will be run on [http://localhost:8080](http://localhost:8080).
### Generating Thrift Code
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this, but if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
To regenerate the code, install thrift-0.9.0 and change directory into Zeppelin source directory. and then run following command
To regenerate the code, install **thrift-0.9.0** and change directory into Zeppelin source directory. and then run following command
```
@ -100,10 +96,10 @@ thrift -out zeppelin-interpreter/src/main/java/ --gen java zeppelin-interpreter/
```
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

37
docs/development/howtocontributewebsite.md
View file

@ -5,62 +5,51 @@ description: "How to contribute (website)"
group: development
---
## IMPORTANT
# Contributing to Apache Zeppelin ( Website )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
## Modifying the website
### Modifying the website
<br />
#### Getting the source code
Website is hosted in 'master' branch under `/docs/` dir.
First of all, you need the website source code. The official location of mirror for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin).
First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git
git clone git://git.apache.org/incubator-zeppelin.git
cd docs
```
<br />
#### Build
To build, you'll need to install some prerequisites.
To build, you'll need to install some prerequisites. Please check 'Build documentation' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build-documentation).
Please check 'Build' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build)
<br />
#### Run website in development mode
While you're modifying website, you'll want to see preview of it.
While you're modifying website, you'll want to see preview of it. Please check 'Run website' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run-website).
Please check 'Run' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run)
You'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser.
You'll be able to access it on localhost:4000 with your webbrowser.
#### Making a Pull Request
<br />
#### Pull request
When you're ready, just make a pull-request.
When you are ready, just make a pull-request.
<br />
### Alternative way
## Alternative way
You can directly edit .md files in `/docs/` dir at github's web interface and make pull-request immediatly.
<br />
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

35
docs/development/writingzeppelininterpreter.md
View file

@ -23,7 +23,7 @@ limitations under the License.
Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need scala interpreter.
Every Interpreter belongs to an InterpreterGroup. InterpreterGroup is a unit of start/stop interpreter.
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/interpreter.png" />
@ -48,6 +48,18 @@ The name of the interpreter is what you later write to identify a paragraph whic
%MyInterpreterName
some interpreter specific code...
```
### Programming Languages for Interpreter
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.
### Install your interpreter binary
Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies.
@ -64,7 +76,7 @@ To configure your interpreter you need to follow these steps:
Property value is comma separated [INTERPRETER\_CLASS\_NAME].
For example,
```
<property>
<name>zeppelin.interpreters</name>
@ -79,7 +91,7 @@ To configure your interpreter you need to follow these steps:
4. In the interpreter page, click the `+Create` button and configure your interpreter properties.
Now you are done and ready to use your interpreter.
Note that the interpreters shipped with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
Note that the interpreters released with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
### Use your interpreter
@ -138,15 +150,24 @@ You can only omit your interpreter group when your interpreter group is selected
codes for myintp2
```
### Examples
Check some interpreters shipped by default.
Checkout some interpreters released with Zeppelin by default.
- [spark](https://github.com/apache/incubator-zeppelin/tree/master/spark)
- [markdown](https://github.com/apache/incubator-zeppelin/tree/master/markdown)
- [shell](https://github.com/apache/incubator-zeppelin/tree/master/shell)
- [hive](https://github.com/apache/incubator-zeppelin/tree/master/hive)
### Contributing a new Interpreter to Zeppelin releases
We welcome contribution to a new interpreter. Please follow these few steps:
- First, check out the general contribution guide [here](./howtocontributewebsite.html).
- Follow the steps in "Make your own Interpreter" section above.
- Add your interpreter as in the "Configure your interpreter" section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/incubator-zeppelin/blob/master/conf/zeppelin-site.xml.template).
- Add tests! They are run by Travis for all changes and it is important that they are self-contained.
- Include your interpreter as a module in [`pom.xml`](https://github.com/apache/incubator-zeppelin/blob/master/pom.xml).
- Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/incubator-zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`).
- Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE).
- Commit your changes and open a Pull Request on the project [Mirror on GitHub](https://github.com/apache/incubator-zeppelin); check to make sure Travis CI build is passing.

185
docs/displaysystem/angular.md
View file

@ -20,32 +20,28 @@ limitations under the License.
{% include JB/setup %}
### Angular (Beta)
## Angular Display System in Zeppelin
Angular display system treats output as an view template of [AngularJS](https://angularjs.org/).
It compiles templates and display inside of Zeppelin.
Angular display system treats output as a view template for [AngularJS](https://angularjs.org/).
It compiles templates and displays them inside of Zeppelin.
Zeppelin provides gateway between your interpreter and your compiled AngularJS view teamplates.
Therefore, you can not only update scope variable from your interpreter but also watch your scope variable in the interpreter, which is JVM process.
Zeppelin provides a gateway between your interpreter and your compiled **AngularJS view** templates.
Therefore, you can not only update scope variables from your interpreter but also watch them in the interpreter, which is JVM process.
### Print AngularJS view
To use angular display system, you should start with `%angular`.
<img src="/assets/themes/zeppelin/img/screenshots/display_angular.png" width="60%" />
Since `name` is not defined, `Hello {{name}}` will display `Hello`.
> **Please Note:** Display system is backend independent.
<br />
#### Print AngularJS view
### Bind / Unbind Variables
To use angular display system, your output should starts with "%angular".
<img src="/assets/themes/zeppelin/img/screenshots/display_angular.png" width=600px />
Through **ZeppelinContext**, you can bind / unbind variables to **AngularJS view**. Currently, it only works in **Spark Interpreter ( scala )**.
Note that display system is backend independent.
Because of variable 'name' is not defined, 'Hello \{\{name\}\}' display 'Hello '.
<br />
#### Bind/Unbind variable
Through ZeppelinContext, you can bind/unbind variable to AngularJS view.
Currently it only works in Spark Interpreter (scala).
```
```scala
// bind my 'object' as angular scope variable 'name' in current notebook.
z.angularBind(String name, Object object)
@ -60,19 +56,17 @@ z.angularUnbindGlobal(String name)
```
In the example, let's bind "world" variable 'name'. Then you can see AngularJs view are updated immediately.
Using the above example, let's bind `world` variable to `name`. Then you can see **AngularJs view** is immediately updated.
<img src="/assets/themes/zeppelin/img/screenshots/display_angular1.png" width=600px />
<img src="/assets/themes/zeppelin/img/screenshots/display_angular1.png" width="60%" />
<br />
#### Watch/Unwatch variable
### Watch / Unwatch Variables
Through ZeppelinContext, you can watch/unwatch variable in AngularJs view.
Through **ZeppelinContext**, you can watch / unwatch variables in **AngularJs view**. Currently, it only works in **Spark Interpreter ( scala )**.
Currently it only works in Spark Interpreter (scala).
```
```scala
// register for angular scope variable 'name' (notebook)
z.angularWatch(String name, (before, after) => { ... })
@ -88,11 +82,138 @@ z.angularUnwatchGlobal(String name)
```
Let's make an button, that increment 'run' variable by 1 when it is clicked.
z.angularBind("run", 0) will initialize 'run' to zero. And then register watcher of 'run'.
Let's make a button. When it is clicked, the value of `run` will be increased 1 by 1.
<img src="/assets/themes/zeppelin/img/screenshots/display_angular2.png" width=600px />
<img src="/assets/themes/zeppelin/img/screenshots/display_angular2.png" width="60%" />
After clicked button, you'll see both 'run' and numWatched are increased by 1
`z.angularBind("run", 0)` will initialize `run` to zero. And then, it will be also applied to `run` in `z.angularWatch()`.
When the button is clicked, you'll see both `run` and `numWatched` are incremented by 1.
<img src="/assets/themes/zeppelin/img/screenshots/display_angular3.png" width=600px />
<img src="/assets/themes/zeppelin/img/screenshots/display_angular3.png" width="60%" />
## Let's make it Simpler and more Intuitive
In this section, we will introduce a simpler and more intuitive way of using **Angular Display System** in Zeppelin.
### How can we use it?
Here are some usages.
#### Import
##### - In notebook scope
```scala
import org.apache.zeppelin.display.angular.notebookscope._
import AngularElem._
```
##### - In paragraph scope
```scala
import org.apache.zeppelin.display.angular.paragraphscope._
import AngularElem._
```
#### Display Element
```scala
// automatically convert to string and print with %angular display system directive in front.
<div><div>.display
```
#### Event Handler
```scala
// on click
<div></div>.onClick(() => {
my callback routine
}).display
// on change
<div></div>.onChange(() => {
my callback routine
}).display
// arbitrary event
<div></div>.onEvent("ng-click", () => {
my callback routine
}).display
```
#### Bind Model
```scala
// bind model
<div></div>.model("myModel").display
// bind model with initial value
<div></div>.model("myModel", initialValue).display
```
#### Interact with Model
```scala
// read model
AngularModel("myModel")()
// update model
AngularModel("myModel", "newValue")
```
<br/>
### Example: Basic Usage
Using the above basic usages, you can apply them like below examples.
#### Display Elements
```scala
<div style="color:blue">
<h4>Hello Angular Display System</h4>
</div>.display
```
#### OnClick Event
```scala
<div class="btn btn-success">
Click me
</div>.onClick{() =>
// callback for button click
}.display
```
#### Bind Model
{% raw %}
```scala
<div>{{{{myModel}}}}</div>.model("myModel", "Initial Value").display
```
{% endraw %}
#### Interact With Model
```scala
// read the value
AngularModel("myModel")()
// update the value
AngularModel("myModel", "New value")
```
<img src="../assets/themes/zeppelin/img/docs-img/basic-usage-angular.png" width="70%">
### Example: String Converter
Using below example, you can convert the lowercase string to uppercase.
{% raw %}
```scala
// clear previously created angular object.
AngularElem.disassociate
val button = <div class="btn btn-success btn-sm">Convert</div>.onClick{() =>
val inputString = AngularModel("input")().toString
AngularModel("title", inputString.toUpperCase)
}
<div>
{ <h4> {{{{title}}}}</h4>.model("title", "Please type text to convert uppercase") }
Your text { <input type="text"></input>.model("input", "") }
{button}
</div>.display
```
{% endraw %}
<img src="../assets/themes/zeppelin/img/docs-img/string-converter-angular.gif" width="70%">

8
docs/index.md
View file

@ -37,7 +37,6 @@ limitations under the License.
<div class="col-md-7"><img class="img-responsive" style="border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/notebook.png" /></div>
</div>
<br />
### Multiple language backend
@ -86,7 +85,6 @@ With simple drag and drop Zeppelin aggeregates the values and display them in pi
</div>
Learn more about Zeppelin's Display system. ( [text](./displaysystem/display.html), [html](./displaysystem/display.html#html), [table](./displaysystem/table.html), [angular](./displaysystem/angular.html) )
<br />
### Dynamic forms
@ -96,7 +94,6 @@ Zeppelin can dynamically create some input forms into your notebook.
Learn more about [Dynamic Forms](./manual/dynamicform.html).
<br />
### Collaboration
@ -114,6 +111,8 @@ This way, you can easily embed it as an iframe inside of your website.</p>
<img class="img-responsive center-block" src="/assets/themes/zeppelin/img/screenshots/publish.png" />
</div>
If you want to learn more about this feature, please visit [this page](./manual/publish.html).
<br />
### 100% Opensource
@ -124,5 +123,4 @@ Join the [Mailing list](./community.html) and report issues on our [Issue tracke
<br />
### Undergoing Incubation
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.

115
docs/install/install.md
View file

@ -1,6 +1,6 @@
---
layout: page
title: "Install Zeppelin"
title: "Zeppelin Installation"
description: ""
group: install
---
@ -21,21 +21,31 @@ limitations under the License.
## From binary package
## Zeppelin Installation
Welcome to your first trial to explore Zeppelin!
Download latest binary package from [Download](../download.html).
In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below.
### Install with Binary Package
## Build from source
If you want to install Zeppelin with latest binary package, please visit [this page](http://zeppelin.incubator.apache.org/download.html).
Check instructions in [README](https://github.com/apache/incubator-zeppelin/blob/master/README.md) to build from source.
### Build from Zeppelin Source
You can also build Zeppelin from the source.
#### Prerequisites for build
* Java 1.7
* Git
* Maven(3.1.x or higher)
* Node.js Package Manager
## Configure
If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/incubator-zeppelin/blob/master/README.md) for the details.
Configuration can be done by both environment variable(conf/zeppelin-env.sh) and java properties(conf/zeppelin-site.xml). If both defined, environment vaiable is used.
<a name="zeppelin-configuration"> </a>
## Zeppelin Configuration
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` and **java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will be used priorly.
<table class="table-configuration">
<tr>
@ -48,7 +58,7 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
<td>ZEPPELIN_PORT</td>
<td>zeppelin.server.port</td>
<td>8080</td>
<td>Zeppelin server port.</td>
<td>Zeppelin server port</td>
</tr>
<tr>
<td>ZEPPELIN_MEM</td>
@ -66,19 +76,25 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
<td>ZEPPELIN_JAVA_OPTS</td>
<td>N/A</td>
<td></td>
<td>JVM Options</td>
<td>JVM options</td>
</tr>
<tr>
<td>ZEPPELIN_ALLOWED_ORIGINS</td>
<td>zeppelin.server.allowed.origins</td>
<td>*</td>
<td>Allows a way to specify a ',' separated list of allowed origins for rest and websockets. i.e. http://localhost:8080</td>
<td>Enables a way to specify a ',' separated list of allowed origins for rest and websockets. <br /> i.e. http://localhost:8080 </td>
</tr>
<tr>
<td>N/A</td>
<td>zeppelin.anonymous.allowed</td>
<td>true</td>
<td>Anonymous user is allowed by default.</td>
</tr>
<tr>
<td>ZEPPELIN_SERVER_CONTEXT_PATH</td>
<td>zeppelin.server.context.path</td>
<td>/</td>
<td>Context Path of the Web Application</td>
<td>A context path of the web application</td>
</tr>
<tr>
<td>ZEPPELIN_SSL</td>
@ -138,37 +154,55 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN</td>
<td>zeppelin.notebook.homescreen</td>
<td></td>
<td>Id of notebook to be displayed in homescreen ex) 2A94M5J1Z</td>
<td>A notebook id displayed in Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE</td>
<td>zeppelin.notebook.homescreen.hide</td>
<td>false</td>
<td>hide homescreen notebook from list when this value set to "true"</td>
<td>This value can be "true" when to hide the notebook id set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
</tr>
<tr>
<td>ZEPPELIN_WAR_TEMPDIR</td>
<td>zeppelin.war.tempdir</td>
<td>webapps</td>
<td>The location of jetty temporary directory.</td>
<td>A location of jetty temporary directory</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_DIR</td>
<td>zeppelin.notebook.dir</td>
<td>notebook</td>
<td>Where notebook file is saved</td>
<td>The root directory where Zeppelin notebook directories are saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_BUCKET</td>
<td>zeppelin.notebook.s3.bucket</td>
<td>zeppelin</td>
<td>Bucket where notebook saved</td>
<td>S3 Bucket where Zeppelin notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_USER</td>
<td>zeppelin.notebook.s3.user</td>
<td>user</td>
<td>User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json</td>
<td>A user name of S3 bucket<br />i.e. <code>bucket/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=&lt;accountName&gt;;AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Share where the Zeppelin notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
<td>zeppelin.notebook.azure.user</td>
<td>user</td>
<td>An optional user name of Azure file share<br />i.e. <code>share/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_STORAGE</td>
@ -183,7 +217,7 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />org.apache.zeppelin.hive.HiveInterpreter<br />
...
</td>
<td>Comma separated interpreter configurations [Class]. First interpreter become a default</td>
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in Zeppelin notebook paragraph. </td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETER_DIR</td>
@ -193,19 +227,16 @@ Configuration can be done by both environment variable(conf/zeppelin-env.sh) and
</tr>
</table>
<br />
You'll also need to configure individual interpreter. Information can be found in 'Interpreter' section in this documentation.
Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation.
[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example.
For example [Spark](../interpreter/spark.html).
<br />
## Start/Stop
## Zeppelin Start / Stop
#### Start Zeppelin
```
bin/zeppelin-daemon.sh start
```
After successful start, visit http://localhost:8080 with your web browser.
After successful start, visit [http://localhost:8080](http://localhost:8080) with your web browser.
#### Stop Zeppelin
@ -213,4 +244,38 @@ After successful start, visit http://localhost:8080 with your web browser.
bin/zeppelin-daemon.sh stop
```
#### Start Zeppelin with a service manager such as upstart
Zeppelin can auto start as a service with an init script, such as services managed by upstart.
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
This example has been tested with Ubuntu Linux.
This also allows the service to be managed with commands such as
`sudo service zeppelin start`
`sudo service zeppelin stop`
`sudo service zeppelin restart`
Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart`
##### zeppelin.conf
```
description "zeppelin"
start on (local-filesystems and net-device-up IFACE!=lo)
stop on shutdown
# Respawn the process on unexpected termination
respawn
# respawn the job up to 7 times within a 5 second period.
# If the job exceeds these values, it will be stopped and marked as failed.
respawn limit 7 5
# zeppelin was installed in /usr/share/zeppelin in this example
chdir /usr/share/zeppelin
exec bin/zeppelin-daemon.sh upstart
```

46
docs/install/virtual_machine.md
View file

@ -22,11 +22,11 @@ limitations under the License.
## Vagrant Virtual Machine for Apache Zeppelin
The Apache Zeppelin distribution includes a scripts directory
Apache Zeppelin distribution includes a scripts directory
`scripts/vagrant/zeppelin-dev`
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras)
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras).
####Installing the required components to launch a virtual machine.
@ -34,19 +34,22 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
If you are running Windows and don't yet have python installed, [install Python 2.7.x](https://www.python.org/downloads/release/python-2710/) first.
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
```
sudo easy_install pip
sudo pip install ansible
ansible --version
```
After then, please check whether it reports **ansible version 1.9.2 or higher**.
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
Thats it!
You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
Thats it ! You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
If you don't wish to build Zeppelin from scratch, run the z-manager installer script while running in the guest VM:
@ -55,18 +58,17 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
```
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning Zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the Zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -74,7 +76,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -117,15 +119,15 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
### [Python Extras](id:pythonextras)
With zeppelin running, Numpy, SciPy, Pandas and Matplotlib will be available. Create a pyspark notebook, and try
With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available. Create a pyspark notebook, and try the below code.
```
```python
%pyspark
import numpy
@ -139,9 +141,9 @@ print "pandas " + pandas.__version__
print "matplotlib " + matplotlib.__version__
```
To Test plotting using matplotlib into a rendered %html SVG image, try
To Test plotting using Matplotlib into a rendered `%html` SVG image, try
```
```python
%pyspark
import matplotlib

132
docs/install/yarn_install.md
View file

@ -20,7 +20,7 @@ limitations under the License.
{% include JB/setup %}
## Introduction
This page describes how to pre-configure a bare metal node, build & configure Zeppelin on it, configure Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop. It also describes steps to configure Spark & Hive interpreter of Zeppelin.
This page describes how to pre-configure a bare metal node, configure Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop. It also describes steps to configure Spark & Hive interpreter of Zeppelin.
## Prepare Node
@ -44,84 +44,16 @@ Its assumed in the rest of the document that zeppelin user is indeed created and
### List of Prerequisites
* CentOS 6.x
* Git
* Java 1.7
* Apache Maven
* Hadoop client.
* Spark.
* CentOS 6.x, Mac OSX, Ubuntu 14.X
* Java 1.7
* Hadoop client
* Spark
* Internet connection is required.
Its assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine. The working directory of all prerequisite pacakges is /home/zeppelin/prerequisites, although any location could be used.
#### Git
Intall latest stable version of Git. This document describes installation of version 2.4.8
```bash
yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel
yum install gcc perl-ExtUtils-MakeMaker
yum remove git
cd /home/zeppelin/prerequisites
wget https://github.com/git/git/archive/v2.4.8.tar.gz
tar xzf git-2.0.4.tar.gz
cd git-2.0.4
make prefix=/home/zeppelin/prerequisites/git all
make prefix=/home/zeppelin/prerequisites/git install
echo "export PATH=$PATH:/home/zeppelin/prerequisites/bin" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
git --version
```
Assuming all the packages are successfully installed, running the version option with git command should display
```bash
git version 2.4.8
```
#### Java
Zeppelin works well with 1.7.x version of Java runtime. Download JDK version 7 and a stable update and follow below instructions to install it.
```bash
cd /home/zeppelin/prerequisites/
#Download JDK 1.7, Assume JDK 7 update 79 is downloaded.
tar -xf jdk-7u79-linux-x64.tar.gz
echo "export JAVA_HOME=/home/zeppelin/prerequisites/jdk1.7.0_79" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
echo $JAVA_HOME
```
Assuming all the packages are successfully installed, echoing JAVA_HOME environment variable should display
```bash
/home/zeppelin/prerequisites/jdk1.7.0_79
```
#### Apache Maven
Download and install a stable version of Maven.
```bash
cd /home/zeppelin/prerequisites/
wget ftp://mirror.reverse.net/pub/apache/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
tar -xf apache-maven-3.3.3-bin.tar.gz
cd apache-maven-3.3.3
export MAVEN_HOME=/home/zeppelin/prerequisites/apache-maven-3.3.3
echo "export PATH=$PATH:/home/zeppelin/prerequisites/apache-maven-3.3.3/bin" >> /home/zeppelin/.bashrc
source /home/zeppelin/.bashrc
mvn -version
```
Assuming all the packages are successfully installed, running the version option with mvn command should display
```bash
Apache Maven 3.3.3 (7994120775791599e205a5524ec3e0dfe41d4a06; 2015-04-22T04:57:37-07:00)
Maven home: /home/zeppelin/prerequisites/apache-maven-3.3.3
Java version: 1.7.0_79, vendor: Oracle Corporation
Java home: /home/zeppelin/prerequisites/jdk1.7.0_79/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "2.6.32-358.el6.x86_64", arch: "amd64", family: "unix"
```
It's assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine.
#### Hadoop client
Zeppelin can work with multiple versions & distributions of Hadoop. A complete list [is available here.](https://github.com/apache/incubator-zeppelin#build) This document assumes Hadoop 2.7.x client libraries including configuration files are installed on Zeppelin node. It also assumes /etc/hadoop/conf contains various Hadoop configuration files. The location of Hadoop configuration files may vary, hence use appropriate location.
Zeppelin can work with multiple versions & distributions of Hadoop. A complete list is available [here](https://github.com/apache/incubator-zeppelin#build). This document assumes Hadoop 2.7.x client libraries including configuration files are installed on Zeppelin node. It also assumes /etc/hadoop/conf contains various Hadoop configuration files. The location of Hadoop configuration files may vary, hence use appropriate location.
```bash
hadoop version
@ -134,32 +66,21 @@ This command was run using /usr/hdp/2.3.1.0-2574/hadoop/lib/hadoop-common-2.7.1.
```
#### Spark
Zeppelin can work with multiple versions Spark. A complete list [is available here.](https://github.com/apache/incubator-zeppelin#build) This document assumes Spark 1.3.1 is installed on Zeppelin node at /home/zeppelin/prerequisites/spark.
Spark is supported out of the box and to take advantage of this, you need to Download appropriate version of Spark binary packages from [Spark Download page](http://spark.apache.org/downloads.html) and unzip it.
Zeppelin can work with multiple versions of Spark. A complete list is available [here](https://github.com/apache/incubator-zeppelin#build).
This document assumes Spark 1.6.0 is installed at /usr/lib/spark.
> Note: Spark should be installed on the same node as Zeppelin.
## Build
> Note: Spark's pre-built package for CDH 4 doesn't support yarn.
Checkout source code from [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
#### Zeppelin
```bash
cd /home/zeppelin/
git clone https://github.com/apache/incubator-zeppelin.git
```
Zeppelin package is available at /home/zeppelin/incubator-zeppelin after the checkout completes.
### Cluster mode
As its assumed Hadoop 2.7.x is installed on the YARN cluster & Spark 1.3.1 is installed on Zeppelin node. Hence appropriate options are chosen to build Zeppelin. This is very important as Zeppelin will bundle corresponding Hadoop & Spark libraries and they must match the ones present on YARN cluster & Zeppelin Spark installation.
Zeppelin is a maven project and hence must be built with Apache Maven.
```bash
cd /home/zeppelin/incubator-zeppelin
mvn clean package -Pspark-1.3 -Dspark.version=1.3.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests
```
Building Zeppelin for first time downloads various dependencies and hence takes few minutes to complete.
Checkout source code from [git://git.apache.org/incubator-zeppelin.git](https://github.com/apache/incubator-zeppelin.git) or download binary package from [Download page](https://zeppelin.incubator.apache.org/download.html).
You can refer [Install](install.html) page for the details.
This document assumes that Zeppelin is located under `/home/zeppelin/incubator-zeppelin`.
## Zeppelin Configuration
Zeppelin configurations needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment XML
Zeppelin configuration needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment shell script.
```bash
cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh
@ -168,9 +89,10 @@ cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppeli
Set the following properties
```bash
export JAVA_HOME=/home/zeppelin/prerequisites/jdk1.7.0_79
export HADOOP_CONF_DIR=/etc/hadoop/conf
export JAVA_HOME="/usr/java/jdk1.7.0_79"
export HADOOP_CONF_DIR="/etc/hadoop/conf"
export ZEPPELIN_JAVA_OPTS="-Dhdp.version=2.3.1.0-2574"
export SPARK_HOME="/usr/lib/spark"
```
As /etc/hadoop/conf contains various configurations of YARN cluster, Zeppelin can now submit Spark/Hive jobs on YARN cluster form its web interface. The value of hdp.version is set to 2.3.1.0-2574. This can be obtained by running the following command
@ -196,7 +118,7 @@ bin/zeppelin-daemon.sh stop
```
## Interpreter
Zeppelin provides to various distributed processing frameworks to process data that ranges from Spark, Hive, Tajo, Ignite and Lens to name a few. This document describes to configure Hive & Spark interpreters.
Zeppelin provides various distributed processing frameworks to process data that ranges from Spark, Hive, Tajo, Ignite and Lens to name a few. This document describes to configure Hive & Spark interpreters.
### Hive
Zeppelin supports Hive interpreter and hence copy hive-site.xml that should be present at /etc/hive/conf to the configuration folder of Zeppelin. Once Zeppelin is built it will have conf folder under /home/zeppelin/incubator-zeppelin.
@ -209,7 +131,7 @@ Once Zeppelin server has started successfully, visit http://[zeppelin-server-hos
Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.
### Spark
Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spark is installed at /home/zeppelin/prerequisites/spark. Look for Spark configrations and click edit button to add the following properties
It was assumed that 1.6.0 version of Spark is installed at /usr/lib/spark. Look for Spark configurations and click edit button to add the following properties
<table class="table-configuration">
<tr>
@ -222,11 +144,6 @@ Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spa
<td>yarn-client</td>
<td>In yarn-client mode, the driver runs in the client process, and the application master is only used for requesting resources from YARN.</td>
</tr>
<tr>
<td>spark.home</td>
<td>/home/zeppelin/prerequisites/spark</td>
<td></td>
</tr>
<tr>
<td>spark.driver.extraJavaOptions</td>
<td>-Dhdp.version=2.3.1.0-2574</td>
@ -237,11 +154,6 @@ Zeppelin was built with Spark 1.3.1 and it was assumed that 1.3.1 version of Spa
<td>-Dhdp.version=2.3.1.0-2574</td>
<td></td>
</tr>
<tr>
<td>spark.yarn.jar</td>
<td>/home/zeppelin/incubator-zeppelin/interpreter/spark/zeppelin-spark-0.6.0-incubating-SNAPSHOT.jar</td>
<td></td>
</tr>
</table>
Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.

914
docs/interpreter/cassandra.md
View file

File diff suppressed because it is too large Load diff

67
docs/interpreter/hbase.md Normal file
View file

@ -0,0 +1,67 @@
---
layout: page
title: "HBase Shell Interpreter"
description: ""
group: manual
---
{% include JB/setup %}
## HBase Shell Interpreter for Apache Zeppelin
[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed.
To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart)
> Note: currently only HBase 1.0.x releases are supported.
## Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
<th>Default</th>
<th>Description</th>
</tr>
<tr>
<td>hbase.home</td>
<td>/usr/lib/hbase</td>
<td>Installation directory of Hbase</td>
</tr>
<tr>
<td>hbase.ruby.sources</td>
<td>lib/ruby</td>
<td>Path to Ruby scripts relative to 'hbase.home'</td>
</tr>
<tr>
<td>hbase.test.mode</td>
<td>false</td>
<td>Disable checks for unit and manual tests</td>
</tr>
</table>
## Enabling the HBase Shell Interpreter
In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**.
## Using the HBase Shell Interpreter
In a paragraph, use `%hbase` to select the **HBase Shell** interpreter and then input all commands. To get the list of available commands, use `help`.
```bash
%hbase
help
```
For example, to create a table
```bash
%hbase
create 'test', 'cf'
```
And then to put data into that table
```bash
%hbase
put 'test', 'row1', 'cf:a', 'value1'
```
For more information on all commands available, refer to [HBase shell commands](https://learnhbase.wordpress.com/2013/03/02/hbase-shell-commands/)

228
docs/interpreter/jdbc.md Normal file
View file

@ -0,0 +1,228 @@
---
layout: page
title: "Generic JDBC Interpreter"
description: "JDBC user guide"
group: manual
---
{% include JB/setup %}
## Generic JDBC Interpreter for Apache Zeppelin
This interpreter lets you create a JDBC connection to any data source, by now it has been tested with:
* Postgres
* MySql
* MariaDB
* Redshift
* Hive
* Apache Drill
* Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver)
If someone else used another database please report how it works to improve functionality.
### Create Interpreter
When create a interpreter by default use PostgreSQL with the next properties:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:postgresql://localhost:5432/</td>
</tr>
<tr>
<td>default.user</td>
<td>gpadmin</td>
</tr>
</table>
It is not necessary to add driver jar to the classpath for PostgreSQL as it is included in Zeppelin.
#### Simple connection
Prior to creating the interpreter it is necessary to add maven coordinate or path of the JDBC driver to the Zeppelin classpath. To do this you must edit dependencies artifact(ex. `mysql:mysql-connector-java:5.1.38`) in interpreter menu as shown:
<div class="row">
<div class="col-md-11">
<img src="../assets/themes/zeppelin/img/docs-img/jdbc-simple-connection-setting.png" />
</div>
</div>
To create the interpreter you need to specify connection parameters as shown in the table.
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>driver name</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc url</td>
</tr>
<tr>
<td>default.user</td>
<td>user name</td>
</tr>
</table>
#### Multiple connections
JDBC interpreter also allows connections to multiple data sources. It is necessary to set a prefix for each connection to reference it in the paragraph in the form of `%jdbc(prefix)`. Before you create the interpreter it is necessary to add each driver's maven coordinates or JDBC driver's jar file path to the Zeppelin classpath. To do this you must edit the dependencies of JDBC interpreter in interpreter menu as following:
<div class="row">
<div class="col-md-11">
<img src="../assets/themes/zeppelin/img/docs-img/jdbc-multi-connection-setting.png" />
</div>
</div>
You can add all the jars you need to make multiple connections into the same JDBC interpreter. To create the interpreter you must specify the parameters. For example we will create two connections to MySQL and Redshift, the respective prefixes are `default` and `redshift`:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>com.mysql.jdbc.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:mysql://localhost:3306/</td>
</tr>
<tr>
<td>default.user</td>
<td>mysql-user</td>
</tr>
<tr>
<td>redshift.driver</td>
<td>com.amazon.redshift.jdbc4.Driver</td>
</tr>
<tr>
<td>redshift.password</td>
<td>********</td>
</tr>
<tr>
<td>redshift.url</td>
<td>jdbc:redshift://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439</td>
</tr>
<tr>
<td>redshift.user</td>
<td>redshift-user</td>
</tr>
</table>
### Bind to Notebook
In the `Notebook` click on the `settings` icon at the top-right corner. Use select/deselect to specify the interpreters to be used in the `Notebook`.
### More Properties
You can modify the interpreter configuration in the `Interpreter` section. The most common properties are as follows, but you can specify other properties that need to be connected.
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
</tr>
<tr>
<td>{prefix}.url</td>
<td>JDBC URL to connect, the URL must include the name of the database </td>
</tr>
<tr>
<td>{prefix}.user</td>
<td>JDBC user name</td>
</tr>
<tr>
<td>{prefix}.password</td>
<td>JDBC password</td>
</tr>
<tr>
<td>{prefix}.driver</td>
<td>JDBC driver name.</td>
</tr>
<tr>
<td>common.max_result</td>
<td>Max number of SQL result to display to prevent the browser overload. This is common properties for all connections</td>
</tr>
</table>
To develop this functionality use this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29). For example if a connection needs a schema parameter, it would have to add the property as follows:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>{prefix}.schema</td>
<td>schema_name</td>
</tr>
</table>
### How to use
#### Reference in paragraph
Start the paragraphs with the `%jdbc`, this will use the `default` prefix for connection. If you want to use other connection you should specify the prefix of it as follows `%jdbc(prefix)`:
```sql
%jdbc
SELECT * FROM db_name;
```
or
```sql
%jdbc(prefix)
SELECT * FROM db_name;
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
%jdbc(prefix)
SELECT name, country, performer
FROM demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
```
### Bugs & Contacts
If you find a bug for this interpreter, please create a [JIRA]( https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN) ticket.

120
docs/interpreter/spark.md
View file

@ -80,9 +80,58 @@ SparkContext, SQLContext, ZeppelinContext are automatically created and exposed
<a name="dependencyloading"> </a>
## Dependency Management
There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties.
There are two ways to load external library in spark interpreter. First is using Interpreter setting menu and second is loading Spark properties.
### 1. Setting Dependencies via Interpreter Setting
Please see [Dependency Management](../manual/dependencymanagement.html) for the details.
### 2. Loading Spark Properties
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
<table class="table-configuration">
<tr>
<th>spark-defaults.conf</th>
<th>SPARK_SUBMIT_OPTIONS</th>
<th>Applicable Interpreter</th>
<th>Description</th>
</tr>
<tr>
<td>spark.jars</td>
<td>--jars</td>
<td>%spark</td>
<td>Comma-separated list of local jars to include on the driver and executor classpaths.</td>
</tr>
<tr>
<td>spark.jars.packages</td>
<td>--packages</td>
<td>%spark</td>
<td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.</td>
</tr>
<tr>
<td>spark.files</td>
<td>--files</td>
<td>%pyspark</td>
<td>Comma-separated list of files to be placed in the working directory of each executor.</td>
</tr>
</table>
> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
Here are few examples:
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
* SPARK_HOME/conf/spark-defaults.conf
spark.jars /path/mylib1.jar,/path/mylib2.jar
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
### 3. Dynamic Dependency Loading via %dep interpreter
> Note: `%dep` interpreter is deprecated since v0.6.0-incubating.
`%dep` interpreter load libraries to `%spark` and `%pyspark` but not to `%spark.sql` interpreter so we recommend you to use first option instead.
### 1. Dynamic Dependency Loading via %dep interpreter
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
* Load libraries recursively from Maven repository
@ -129,49 +178,6 @@ z.load("groupId:artifactId:version").exclude("groupId:*")
z.load("groupId:artifactId:version").local()
```
### 2. Loading Spark Properties
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
<table class="table-configuration">
<tr>
<th>spark-defaults.conf</th>
<th>SPARK_SUBMIT_OPTIONS</th>
<th>Applicable Interpreter</th>
<th>Description</th>
</tr>
<tr>
<td>spark.jars</td>
<td>--jars</td>
<td>%spark</td>
<td>Comma-separated list of local jars to include on the driver and executor classpaths.</td>
</tr>
<tr>
<td>spark.jars.packages</td>
<td>--packages</td>
<td>%spark</td>
<td>Comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.</td>
</tr>
<tr>
<td>spark.files</td>
<td>--files</td>
<td>%pyspark</td>
<td>Comma-separated list of files to be placed in the working directory of each executor.</td>
</tr>
</table>
> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
Here are few examples:
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
* SPARK_HOME/conf/spark-defaults.conf
spark.jars /path/mylib1.jar,/path/mylib2.jar
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
## ZeppelinContext
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
@ -256,3 +262,27 @@ select * from ${table=defaultTableName} where text like '%${search}%'
```
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
## Setting up Zeppelin with Kerberos
Logical setup with Zeppelin, Kerberos Distribution Center (KDC), and Spark on YARN:
<img src="../assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png">
####Configuration Setup
1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf.
This is to make the server communicate with KDC.
2. Set SPARK\_HOME in [ZEPPELIN\_HOME]/conf/zeppelin-env.sh to use spark-submit
( Additionally, you might have to set “export HADOOP\_CONF\_DIR=/etc/hadoop/conf” )
3. Add the two properties below to spark configuration ( [SPARK_HOME]/conf/spark-defaults.conf ):
spark.yarn.principal
spark.yarn.keytab
> **NOTE:** If you do not have access to the above spark-defaults.conf file, optionally, you may add the lines to the Spark Interpreter through the Interpreter tab in the Zeppelin UI.
4. That's it. Play with Zeppelin !

74
docs/manual/dependencymanagement.md Normal file
View file

@ -0,0 +1,74 @@
---
layout: page
title: "Dependnecy Management"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## Dependency Management for Interpreter
You can include external libraries to interpreter by setting dependencies in interpreter menu.
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs in this menu.
* Load libraries recursively from Maven repository
* Load libraries from local filesystem
* Add additional maven repository
* Automatically add libraries to SparkCluster
<hr>
<div class="row">
<div class="col-md-6">
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png" />
</a>
</div>
<div class="col-md-6" style="padding-top:30px">
<b> Load Dependencies to Interpreter </b>
<br /><br />
<ol>
<li> Click 'Interpreter' menu in navigation bar. </li>
<li> Click 'edit' button of the interpreter which you want to load dependencies to. </li>
<li> Fill artifact and exclude field to your needs.
You can enter not only groupId:artifactId:version but also local file in artifact field. </li>
<li> Press 'Save' to restart the interpreter with loaded libraries. </li>
</ol>
</div>
</div>
<hr>
<div class="row">
<div class="col-md-6">
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png" />
</a>
<a data-lightbox="compiler" href="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png">
<img class="img-responsive" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png" />
</a>
</div>
<div class="col-md-6" style="padding-top:30px">
<b> Add repository for dependency resolving </b>
<br /><br />
<ol>
<li> Press <i class="fa fa-cog"></i> icon in 'Interpreter' menu on the top right side.
It will show you available repository lists.</li>
<li> If you need to resolve dependencies from other than central maven repository or
local ~/.m2 repository, hit <i class="fa fa-plus"></i> icon next to repository lists. </li>
<li> Fill out the form and click 'Add' button, then you will be able to see that new repository is added. </li>
</ol>
</div>
</div>

16
docs/manual/interpreters.md
View file

@ -25,7 +25,7 @@ The concept of Zeppelin interpreter allows any language/data-processing-backend
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, Hive, Markdown, Shell and so on.
## What is Zeppelin interpreter?
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use scala code in Zeppelin, you need `%spark` interpreter.
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use Scala code in Zeppelin, you need `%spark` interpreter.
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
@ -38,21 +38,9 @@ Zeppelin interpreter setting is the configuration of a given interpreter on Zepp
## What is Zeppelin Interpreter Group?
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, spark interpreter group is including Spark support, pySpark,
SparkSQL and the dependency loader.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, SparkSQL and the dependency loader.
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
## Programming Languages for Interpreter
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.

48
docs/manual/publish.md Normal file
View file

@ -0,0 +1,48 @@
---
layout: page
title: "Publish your Paragraph"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## How can you publish your paragraph ?
Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website.
It's very straightforward. Just use `<iframe>` tag in your page.
> **Warning**: Please use this feature with caution and in a trusted environment only, as Zeppelin entire Webapp could be accessible for whoever visits your website.
### Copy a Paragraph Link
A first step to publish your paragraph result is **Copy a Paragraph Link**.
* After running a paragraph in your Zeppelin notebook, click a gear button located on the right side. Then, click **Link this Paragraph** menu like below image.
<center><img src="../assets/themes/zeppelin/img/docs-img/link-the-paragraph.png" height="100%" width="100%"></center>
* Just copy the provided link.
<center><img src="../assets/themes/zeppelin/img/docs-img/copy-the-link.png" height="100%" width="100%"></center>
### Embed the Paragraph to Your Website
For publishing the copied paragraph, you may use `<iframe>` tag in your website page.
For example,
```
<iframe src="http://< ip-address >:< port >/#/notebook/2B3QSZTKR/paragraph/...?asIframe" height="" width="" ></iframe>
```
Finally, you can show off your beautiful visualization results in your website.
<center><img src="../assets/themes/zeppelin/img/docs-img/your-website.png" height="90%" width="90%"></center>
> **Note**: To embed the paragraph in a website, Zeppelin needs to be reachable by that website.

72
docs/manual/shiroauthentication.md Normal file
View file

@ -0,0 +1,72 @@
---
layout: page
title: "Shiro Security for Apache Zeppelin"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Shiro authentication for Apache Zeppelin
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
When you connect to Apache Zeppelin, you will be asked to enter your credentials. Once you logged in, then you have access to all notes including other user's notes.
## Security Setup
You can setup **Zeppelin notebook authentication** in some simple steps.
####1. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
The default status of them is
```
/** = anon
#/** = authcBasic
```
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
```
#/** = anon
/** = authcBasic
```
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
####2. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
####3. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
```
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
```
admin = password1
user1 = password2
user2 = password3
```
Those combinations are defined in the `conf/shiro.ini` file.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/incubator-zeppelin/blob/master/SECURITY-README.md).

104
docs/rest-api/rest-interpreter.md
View file

@ -151,7 +151,8 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
]
],
"dependencies": []
},
{
"id": "2AY6GV7Q3",
@ -170,6 +171,11 @@ limitations under the License.
"class": "org.apache.zeppelin.spark.SparkSqlInterpreter",
"name": "sql"
}
],
"dependencies": [
{
"groupArtifactVersion": "com.databricks:spark-csv_2.10:1.3.0"
}
]
}
]
@ -219,6 +225,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
],
"dependencies": [
{
"groupArtifactVersion": "groupId:artifactId:version",
"exclusions": "groupId:artifactId"
}
]
}
</pre>
@ -243,6 +255,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
],
"dependencies": [
{
"groupArtifactVersion": "groupId:artifactId:version",
"exclusions": "groupId:artifactId"
}
]
}
}
@ -292,6 +310,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
],
"dependencies": [
{
"groupArtifactVersion": "groupId:artifactId:version",
"exclusions": "groupId:artifactId"
}
]
}
</pre>
@ -316,6 +340,12 @@ limitations under the License.
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
],
"dependencies": [
{
"groupArtifactVersion": "groupId:artifactId:version",
"exclusions": "groupId:artifactId"
}
]
}
}
@ -391,3 +421,75 @@ limitations under the License.
</td>
</tr>
</table>
<br/>
### 6. Add repository for dependency resolving
<table class="table-configuration">
<col width="200">
<tr>
<th>Add new repository for dependency loader</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```POST``` method adds new repository.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository```</td>
</tr>
<tr>
<td>Success code</td>
<td>201</td>
</tr>
<tr>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td>Sample JSON input</td>
<td>
<pre>
{
"id": "securecentral",
"url": "https://repo1.maven.org/maven2",
"snapshot": false
}
</pre>
</td>
</tr>
<tr>
<td>Sample JSON response</td>
<td>
<code>{"status":"OK"}</code>
</td>
</tr>
</table>
<br/>
### 7. Delete repository for dependency resolving
<table class="table-configuration">
<col width="200">
<tr>
<th>Delete repository for dependency loader</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```DELETE``` method delete repository with given id.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository/[repository ID]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td>Fail code</td>
<td> 500 </td>
</tr>
</table>

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class FlinkInterpreterTest {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null, null);
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
}
@AfterClass

176
hbase/pom.xml Normal file
View file

@ -0,0 +1,176 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-hbase</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: HBase interpreter</name>
<url>http://www.apache.org</url>
<properties>
<hbase.hbase.version>1.0.0</hbase.hbase.version>
<hbase.hadoop.version>2.6.0</hbase.hadoop.version>
<jruby.version>1.6.8</jruby.version>
<protobuf.version>2.5.0</protobuf.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby-complete</artifactId>
<version>${jruby.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>${hbase.hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${hbase.hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
<version>${hbase.hbase.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.hbase.version}</version>
</dependency>
<dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>2.12.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

158
hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java Normal file
View file

@ -0,0 +1,158 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.hbase;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.jruby.embed.LocalContextScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.jruby.embed.ScriptingContainer;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
/**
* Support for Hbase Shell. All the commands documented here
* http://hbase.apache.org/book.html#shell is supported.
*
* Requirements:
* Hbase Shell should be installed on the same machine. To be more specific, the following dir.
* should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
* Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure
* that the client is configured properly.
*
* The interpreter takes 3 config parameters:
* hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/
* hbase.ruby.sources: Dir where shell ruby code is installed.
* Path is relative to hbase.home. Default: lib/ruby
* hbase.irb.load: (Testing only) Default is true.
* Whether to load irb in the interpreter.
*/
public class HbaseInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
private ScriptingContainer scriptingContainer;
private StringWriter writer;
static {
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
.add("hbase.ruby.sources", "lib/ruby",
"Path to Ruby scripts relative to 'hbase.home'")
.add("hbase.test.mode", "false", "Disable checks for unit and manual tests")
.build());
}
public HbaseInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON);
this.writer = new StringWriter();
scriptingContainer.setOutput(this.writer);
if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) {
String hbase_home = getProperty("hbase.home");
String ruby_src = getProperty("hbase.ruby.sources");
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
logger.info("Home:" + hbase_home);
logger.info("Ruby Src:" + ruby_src);
File f = abs_ruby_src.toFile();
if (!f.exists() || !f.isDirectory()) {
throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src
+ "'");
}
logger.info("Absolute Ruby Source:" + abs_ruby_src.toString());
// hirb.rb:41 requires the following system property to be set.
Properties sysProps = System.getProperties();
sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString());
Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb");
try {
FileInputStream fis = new FileInputStream(abs_hirb_path.toFile());
this.scriptingContainer.runScriptlet(fis, "hirb.rb");
fis.close();
} catch (IOException e) {
throw new InterpreterException(e.getCause());
}
}
}
@Override
public void close() {
if (this.scriptingContainer != null) {
this.scriptingContainer.terminate();
}
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) {
try {
logger.info(cmd);
this.writer.getBuffer().setLength(0);
this.scriptingContainer.runScriptlet(cmd);
this.writer.flush();
logger.debug(writer.toString());
return new InterpreterResult(InterpreterResult.Code.SUCCESS, writer.getBuffer().toString());
} catch (Throwable t) {
logger.error("Can not run '" + cmd + "'", t);
return new InterpreterResult(InterpreterResult.Code.ERROR, t.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
HbaseInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

75
hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.hbase;
import org.apache.log4j.BasicConfigurator;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertEquals;
/**
* Tests for HBase Interpreter
*/
public class HbaseInterpreterTest {
private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class);
private static HbaseInterpreter hbaseInterpreter;
@BeforeClass
public static void setUp() throws NullPointerException {
BasicConfigurator.configure();
Properties properties = new Properties();
properties.put("hbase.home", "");
properties.put("hbase.ruby.sources", "");
properties.put("hbase.test.mode", "true");
hbaseInterpreter = new HbaseInterpreter(properties);
hbaseInterpreter.open();
}
@Test
public void newObject() {
assertThat(hbaseInterpreter, notNullValue());
}
@Test
public void putsTest() {
InterpreterResult result = hbaseInterpreter.interpret("puts \"Hello World\"", null);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(result.type(), InterpreterResult.Type.TEXT);
assertEquals("Hello World\n", result.message());
}
public void putsLoadPath() {
InterpreterResult result = hbaseInterpreter.interpret("require 'two_power'; puts twoToThePowerOf(4)", null);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(result.type(), InterpreterResult.Type.TEXT);
assertEquals("16\n", result.message());
}
@Test
public void testException() {
InterpreterResult result = hbaseInterpreter.interpret("plot practical joke", null);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
}
}

37
hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java
View file

@ -169,6 +169,9 @@ public class HiveInterpreter extends Interpreter {
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
}
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
@ -203,6 +206,10 @@ public class HiveInterpreter extends Interpreter {
} else {
connection = getConnection(propertyKey);
}
if (connection == null) {
return null;
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
@ -232,6 +239,10 @@ public class HiveInterpreter extends Interpreter {
Statement statement = getStatement(propertyKey, paragraphId);
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
statement.setMaxRows(getMaxResult());
StringBuilder msg;
@ -315,10 +326,8 @@ public class HiveInterpreter extends Interpreter {
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
if (null != propertyKey && !propertyKey.equals(DEFAULT_KEY)) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
@ -334,17 +343,19 @@ public class HiveInterpreter extends Interpreter {
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
boolean firstLineIndex = cmd.startsWith("(");
if (firstLineIndex) {
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1) {
return cmd.substring(configStartIndex + 1, configLastIndex);
} else {
return null;
}
} else {
return DEFAULT_KEY;
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override

60
hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java
View file

@ -66,7 +66,53 @@ public class HiveInterpreterTest {
@After
public void tearDown() throws Exception {
}
@Test
public void testForParsePropertyKey() throws IOException {
HiveInterpreter t = new HiveInterpreter(new Properties());
assertEquals(t.getPropertyKey("(fake) select max(cant) from test_table where id >= 2452640"),
"fake");
assertEquals(t.getPropertyKey("() select max(cant) from test_table where id >= 2452640"),
"");
assertEquals(t.getPropertyKey(")fake( select max(cant) from test_table where id >= 2452640"),
"default");
// when you use a %hive(prefix1), prefix1 is the propertyKey as form part of the cmd string
assertEquals(t.getPropertyKey("(prefix1)\n select max(cant) from test_table where id >= 2452640"),
"prefix1");
assertEquals(t.getPropertyKey("(prefix2) select max(cant) from test_table where id >= 2452640"),
"prefix2");
// when you use a %hive, prefix is the default
assertEquals(t.getPropertyKey("select max(cant) from test_table where id >= 2452640"),
"default");
}
@Test
public void testForMapPrefix() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
String sqlQuery = "(fake) select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
// if prefix not found return ERROR and Prefix not found.
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals("Prefix not found.", interpreterResult.message());
}
@Test
public void readTest() throws IOException {
Properties properties = new Properties();
@ -79,9 +125,9 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null,null)).message().contains("SCHEMA_NAME"));
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message().contains("SCHEMA_NAME"));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message());
}
@Test
@ -101,7 +147,7 @@ public class HiveInterpreterTest {
t.open();
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null)).message());
}
@Test
@ -117,13 +163,13 @@ public class HiveInterpreterTest {
t.open();
InterpreterResult interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
t.getConnection("default").close();
interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@ -139,7 +185,7 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null);
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null, null, null);
//simple select test
InterpreterResult result = t.interpret("select * from test_table", interpreterContext);
@ -193,4 +239,4 @@ public class HiveInterpreterTest {
assertEquals("get key of default", "default", hi.getPropertyKey(testCommand));
hi.close();
}
}
}

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -44,7 +44,7 @@ public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;

50
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@ -172,6 +173,9 @@ public class JDBCInterpreter extends Interpreter {
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
}
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
@ -206,6 +210,10 @@ public class JDBCInterpreter extends Interpreter {
} else {
connection = getConnection(propertyKey);
}
if (connection == null) {
return null;
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
@ -260,6 +268,10 @@ public class JDBCInterpreter extends Interpreter {
try {
Statement statement = getStatement(propertyKey, paragraphId);
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
statement.setMaxRows(getMaxResult());
StringBuilder msg = null;
@ -293,7 +305,15 @@ public class JDBCInterpreter extends Interpreter {
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
msg.append(replaceReservedChars(isTableType, resultSet.getString(i)));
Object resultObject;
String resultValue;
resultObject = resultSet.getObject(i);
if (resultObject == null) {
resultValue = "null";
} else {
resultValue = resultSet.getString(i);
}
msg.append(replaceReservedChars(isTableType, resultValue));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
@ -344,12 +364,10 @@ public class JDBCInterpreter extends Interpreter {
logger.info("Run SQL command '{}'", cmd);
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
if (null != propertyKey && !propertyKey.equals(DEFAULT_KEY)) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
@ -371,17 +389,19 @@ public class JDBCInterpreter extends Interpreter {
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
boolean firstLineIndex = cmd.startsWith("(");
if (firstLineIndex) {
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1) {
return cmd.substring(configStartIndex + 1, configLastIndex);
} else {
return null;
}
} else {
return DEFAULT_KEY;
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override

87
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -26,10 +26,7 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.*;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -64,11 +61,59 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
statement.execute(
"DROP TABLE IF EXISTS test_table; " +
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
statement.execute(
"insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name');"
);
PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
}
@Test
public void testForParsePropertyKey() throws IOException {
JDBCInterpreter t = new JDBCInterpreter(new Properties());
assertEquals(t.getPropertyKey("(fake) select max(cant) from test_table where id >= 2452640"),
"fake");
assertEquals(t.getPropertyKey("() select max(cant) from test_table where id >= 2452640"),
"");
assertEquals(t.getPropertyKey(")fake( select max(cant) from test_table where id >= 2452640"),
"default");
// when you use a %jdbc(prefix1), prefix1 is the propertyKey as form part of the cmd string
assertEquals(t.getPropertyKey("(prefix1)\n select max(cant) from test_table where id >= 2452640"),
"prefix1");
assertEquals(t.getPropertyKey("(prefix2) select max(cant) from test_table where id >= 2452640"),
"prefix2");
// when you use a %jdbc, prefix is the default
assertEquals(t.getPropertyKey("select max(cant) from test_table where id >= 2452640"),
"default");
}
@Test
public void testForMapPrefix() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "(fake) select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
// if prefix not found return ERROR and Prefix not found.
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals("Prefix not found.", interpreterResult.message());
}
@Test
public void testDefaultProperties() throws SQLException {
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(new Properties());
@ -92,15 +137,37 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table";
String sqlQuery = "select * from test_table WHERE ID in ('a', 'b')";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@Test
public void testSelectQueryWithNull() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table WHERE ID = 'c'";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\nc\tnull\n", interpreterResult.message());
}
@Test
public void testSelectQueryMaxResult() throws SQLException, IOException {
@ -116,7 +183,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());

11
lens/pom.xml
View file

@ -226,15 +226,4 @@
</plugins>
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
</profiles>
</project>

12
pom.xml
View file

@ -86,12 +86,14 @@
<modules>
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>zeppelin-display</module>
<module>spark-dependencies</module>
<module>spark</module>
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>hive</module>
<module>hbase</module>
<module>phoenix</module>
<module>postgresql</module>
<module>jdbc</module>
@ -633,6 +635,16 @@
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<!-- Geode can be enabled by -Pgeode. see https://issues.apache.org/jira/browse/ZEPPELIN-375 -->
<profile>
<id>geode</id>

5
scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java
View file

@ -26,6 +26,7 @@ import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@ -62,9 +63,9 @@ public class ScaldingInterpreterTest {
}
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), null);
}

14
scripts/vagrant/zeppelin-dev/README.md
View file

@ -24,9 +24,9 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
@ -45,7 +45,7 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
@ -53,7 +53,7 @@ Cloning the project again may seem counter intuitive, since this script likley o
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -61,7 +61,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -104,7 +104,7 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..

2
scripts/vagrant/zeppelin-dev/show-instructions.sh
View file

@ -18,7 +18,7 @@ echo '# Post vagrant up instructions.'
echo '# From your host machine,'
echo '# git clone the incubator-zeppelin branch into this directory'
echo
echo 'git clone https://github.com/apache/incubator-zeppelin.git'
echo 'git clone git://git.apache.org/incubator-zeppelin.git'
echo
echo '# Cloning the project again may seem counter intuitive, since this script'
echo '# originated from the project repository. Consider copying just the vagrant/zeppelin-dev'

6
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -72,7 +72,7 @@ public class ShellInterpreter extends Interpreter {
DefaultExecutor executor = new DefaultExecutor();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream));
executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeOut));
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
@ -82,7 +82,7 @@ public class ShellInterpreter extends Interpreter {
int exitVal = executor.execute(cmdLine);
logger.info("Paragraph " + contextInterpreter.getParagraphId()
+ "return with exit value: " + exitVal);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString());
return new InterpreterResult(InterpreterResult.Code.SUCCESS, null);
} catch (ExecuteException e) {
int exitValue = e.getExitValue();
logger.error("Can not run " + cmd, e);
@ -94,7 +94,7 @@ public class ShellInterpreter extends Interpreter {
logger.info("The paragraph " + contextInterpreter.getParagraphId()
+ " stopped executing: " + msg);
}
msg += "Exitvalue: " + exitValue;
msg += "ExitValue: " + exitValue;
return new InterpreterResult(code, msg);
} catch (IOException e) {
logger.error("Can not run " + cmd, e);

13
spark-dependencies/pom.xml
View file

@ -339,16 +339,6 @@
</dependencies>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<profile>
<id>spark-1.1</id>
<dependencies>
@ -494,13 +484,14 @@
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
<protobuf.version>2.5.0</protobuf.version>
<guava.version>16.0.1</guava.version>
</properties>
<dependencies>
<dependency>
<groupId>com.datastax.spark</groupId>
<artifactId>spark-cassandra-connector_${scala.binary.version}</artifactId>
<version>1.5.0-M2</version>
<version>1.5.0</version>
<exclusions>
<exclusion>
<groupId>org.joda</groupId>

6
spark/pom.xml
View file

@ -52,6 +52,12 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-display</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>

26
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -59,9 +59,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import py4j.GatewayServer;
@ -124,12 +121,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
// load libraries from Dependency Interpreter
URL [] urls = new URL[0];
List<URL> urlList = new LinkedList<URL>();
if (depInterpreter != null) {
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
List<URL> urlList = new LinkedList<URL>();
if (files != null) {
for (File f : files) {
try {
@ -138,12 +135,29 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
logger.error("Error", e);
}
}
urls = urlList.toArray(urls);
}
}
}
String localRepo = getProperty("zeppelin.interpreter.localRepo");
if (localRepo != null) {
File localRepoDir = new File(localRepo);
if (localRepoDir.exists()) {
File[] files = localRepoDir.listFiles();
if (files != null) {
for (File f : files) {
try {
urlList.add(f.toURI().toURL());
} catch (MalformedURLException e) {
logger.error("Error", e);
}
}
}
}
}
urls = urlList.toArray(urls);
ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader newCl = new URLClassLoader(urls, oldCl);

57
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -87,7 +87,9 @@ public class SparkInterpreter extends Interpreter {
"spark",
SparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("spark.app.name", "Zeppelin", "The name of spark application.")
.add("spark.app.name",
getSystemDefault("SPARK_APP_NAME", "spark.app.name", "Zeppelin"),
"The name of spark application.")
.add("master",
getSystemDefault("MASTER", "spark.master", "local[*]"),
"Spark master uri. ex) spark://masterhost:7077")
@ -324,15 +326,24 @@ public class SparkInterpreter extends Interpreter {
}
}
pythonLibUris.trimToSize();
if (pythonLibs.length == pythonLibUris.size()) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
// Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
// when spark version is less than or equal to 1.4.1
if (pythonLibUris.size() == 2) {
try {
String confValue = conf.get("spark.yarn.dist.files");
conf.set("spark.yarn.dist.files", confValue + "," + Joiner.on(",").join(pythonLibUris));
} catch (NoSuchElementException e) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
}
if (!useSparkSubmit()) {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
}
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
}
// Distributes needed libraries to workers.
// Distributes needed libraries to workers
// when spark version is greater than or equal to 1.5.0
if (getProperty("master").equals("yarn-client")) {
conf.set("spark.yarn.isPython", "true");
}
@ -438,6 +449,23 @@ public class SparkInterpreter extends Interpreter {
}
}
// add dependency from local repo
String localRepo = getProperty("zeppelin.interpreter.localRepo");
if (localRepo != null) {
File localRepoDir = new File(localRepo);
if (localRepoDir.exists()) {
File[] files = localRepoDir.listFiles();
if (files != null) {
for (File f : files) {
if (classpath.length() > 0) {
classpath += File.pathSeparator;
}
classpath += f.getAbsolutePath();
}
}
}
}
pathSettings.v_$eq(classpath);
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
@ -529,7 +557,7 @@ public class SparkInterpreter extends Interpreter {
throw new InterpreterException(e);
}
// add jar
// add jar from DepInterpreter
if (depInterpreter != null) {
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
@ -547,6 +575,25 @@ public class SparkInterpreter extends Interpreter {
}
}
}
// add jar from local repo
if (localRepo != null) {
File localRepoDir = new File(localRepo);
if (localRepoDir.exists()) {
File[] files = localRepoDir.listFiles();
if (files != null) {
for (File f : files) {
if (f.getName().toLowerCase().endsWith(".jar")) {
sc.addJar(f.getAbsolutePath());
logger.info("sc.addJar(" + f.getAbsolutePath() + ")");
} else {
sc.addFile(f.getAbsolutePath());
logger.info("sc.addFile(" + f.getAbsolutePath() + ")");
}
}
}
}
}
}
private List<File> currentClassPath() {

215
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -17,23 +17,17 @@
package org.apache.zeppelin.spark;
import static scala.collection.JavaConversions.asJavaCollection;
import static scala.collection.JavaConversions.asJavaIterable;
import static scala.collection.JavaConversions.collectionAsScalaIterable;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SQLContext.QueryExecution;
import org.apache.spark.sql.catalyst.expressions.Attribute;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.zeppelin.display.AngularObject;
@ -45,15 +39,17 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourceSet;
import scala.Tuple2;
import scala.Unit;
import scala.collection.Iterable;
/**
* Spark context for zeppelin.
*/
public class ZeppelinContext extends HashMap<String, Object> {
public class ZeppelinContext {
private SparkDependencyResolver dep;
private InterpreterContext interpreterContext;
private int maxResult;
@ -74,127 +70,6 @@ public class ZeppelinContext extends HashMap<String, Object> {
public HiveContext hiveContext;
private GUI gui;
/**
* Load dependency for interpreter and runtime (driver).
* And distribute them to spark cluster (sc.add())
*
* @param artifact "group:artifact:version" or file path like "/somepath/your.jar"
* @return
* @throws Exception
*/
public Iterable<String> load(String artifact) throws Exception {
return collectionAsScalaIterable(dep.load(artifact, true));
}
/**
* Load dependency and it's transitive dependencies for interpreter and runtime (driver).
* And distribute them to spark cluster (sc.add())
*
* @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
* @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
* @return
* @throws Exception
*/
public Iterable<String> load(String artifact, scala.collection.Iterable<String> excludes)
throws Exception {
return collectionAsScalaIterable(
dep.load(artifact,
asJavaCollection(excludes),
true));
}
/**
* Load dependency and it's transitive dependencies for interpreter and runtime (driver).
* And distribute them to spark cluster (sc.add())
*
* @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
* @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
* @return
* @throws Exception
*/
public Iterable<String> load(String artifact, Collection<String> excludes) throws Exception {
return collectionAsScalaIterable(dep.load(artifact, excludes, true));
}
/**
* Load dependency for interpreter and runtime, and then add to sparkContext.
* But not adding them to spark cluster
*
* @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
* @return
* @throws Exception
*/
public Iterable<String> loadLocal(String artifact) throws Exception {
return collectionAsScalaIterable(dep.load(artifact, false));
}
/**
* Load dependency and it's transitive dependencies and then add to sparkContext.
* But not adding them to spark cluster
*
* @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
* @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
* @return
* @throws Exception
*/
public Iterable<String> loadLocal(String artifact,
scala.collection.Iterable<String> excludes) throws Exception {
return collectionAsScalaIterable(dep.load(artifact,
asJavaCollection(excludes), false));
}
/**
* Load dependency and it's transitive dependencies and then add to sparkContext.
* But not adding them to spark cluster
*
* @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar"
* @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string.
* @return
* @throws Exception
*/
public Iterable<String> loadLocal(String artifact, Collection<String> excludes)
throws Exception {
return collectionAsScalaIterable(dep.load(artifact, excludes, false));
}
/**
* Add maven repository
*
* @param id id of repository ex) oss, local, snapshot
* @param url url of repository. supported protocol : file, http, https
*/
public void addRepo(String id, String url) {
addRepo(id, url, false);
}
/**
* Add maven repository
*
* @param id id of repository
* @param url url of repository. supported protocol : file, http, https
* @param snapshot true if it is snapshot repository
*/
public void addRepo(String id, String url, boolean snapshot) {
dep.addRepo(id, url, snapshot);
}
/**
* Remove maven repository by id
* @param id id of repository
*/
public void removeRepo(String id){
dep.delRepo(id);
}
/**
* Load dependency only interpreter.
*
* @param name
* @return
*/
public Object input(String name) {
return input(name, "");
}
@ -322,16 +197,14 @@ public class ZeppelinContext extends HashMap<String, Object> {
throw new InterpreterException(e);
}
String msg = null;
StringBuilder msg = new StringBuilder();
msg.append("%table ");
for (Attribute col : columns) {
if (msg == null) {
msg = col.name();
} else {
msg += "\t" + col.name();
}
msg.append(col.name() + "\t");
}
msg += "\n";
String trim = msg.toString().trim();
msg = new StringBuilder(trim);
msg.append("\n");
// ArrayType, BinaryType, BooleanType, ByteType, DecimalType, DoubleType, DynamicType,
// FloatType, FractionalType, IntegerType, IntegralType, LongType, MapType, NativeType,
@ -345,15 +218,15 @@ public class ZeppelinContext extends HashMap<String, Object> {
for (int i = 0; i < columns.size(); i++) {
if (!(Boolean) isNullAt.invoke(row, i)) {
msg += apply.invoke(row, i).toString();
msg.append(apply.invoke(row, i).toString());
} else {
msg += "null";
msg.append("null");
}
if (i != columns.size() - 1) {
msg += "\t";
msg.append("\t");
}
}
msg += "\n";
msg.append("\n");
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@ -361,10 +234,10 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
if (rows.length > maxResult) {
msg += "\n<font color=red>Results are limited by " + maxResult + ".</font>";
msg.append("\n<font color=red>Results are limited by " + maxResult + ".</font>");
}
sc.clearJobGroup();
return "%table " + msg;
return msg.toString();
}
/**
@ -754,4 +627,60 @@ public class ZeppelinContext extends HashMap<String, Object> {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
registry.remove(name, noteId, null);
}
/**
* Add object into resource pool
* @param name
* @param value
*/
public void put(String name, Object value) {
ResourcePool resourcePool = interpreterContext.getResourcePool();
resourcePool.put(name, value);
}
/**
* Get object from resource pool
* Search local process first and then the other processes
* @param name
* @return null if resource not found
*/
public Object get(String name) {
ResourcePool resourcePool = interpreterContext.getResourcePool();
Resource resource = resourcePool.get(name);
if (resource != null) {
return resource.get();
} else {
return null;
}
}
/**
* Remove object from resourcePool
* @param name
*/
public void remove(String name) {
ResourcePool resourcePool = interpreterContext.getResourcePool();
resourcePool.remove(name);
}
/**
* Check if resource pool has the object
* @param name
* @return
*/
public boolean containsKey(String name) {
ResourcePool resourcePool = interpreterContext.getResourcePool();
Resource resource = resourcePool.get(name);
return resource != null;
}
/**
* Get all resources
*/
public ResourceSet getAll() {
ResourcePool resourcePool = interpreterContext.getResourcePool();
return resourcePool.getAll();
}
}

10
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
View file

@ -42,6 +42,8 @@ import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
import scala.Console;
/**
*
@ -64,6 +66,8 @@ public class SparkDependencyContext {
}
public Dependency load(String lib) {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Load dependency through GUI interpreter menu instead.");
Dependency dep = new Dependency(lib);
if (dependencies.contains(dep)) {
@ -74,12 +78,16 @@ public class SparkDependencyContext {
}
public Repository addRepo(String name) {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Add repository through GUI interpreter menu instead.");
Repository rep = new Repository(name);
repositories.add(rep);
return rep;
}
public void reset() {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Remove dependencies and repositories through GUI interpreter menu instead.");
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
@ -156,7 +164,7 @@ public class SparkDependencyContext {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {

10
spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java
View file

@ -25,11 +25,9 @@ import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.After;
import org.junit.Before;
@ -58,8 +56,10 @@ public class DepInterpreterTest {
intpGroup.add(dep);
dep.setInterpreterGroup(intpGroup);
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(), null);
}

37
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -27,6 +27,7 @@ import java.util.Properties;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
@ -76,21 +77,23 @@ public class SparkInterpreterTest {
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
}));
}
@After
@ -172,16 +175,6 @@ public class SparkInterpreterTest {
assertEquals(Code.ERROR, result.code());
}
@Test
public void testZContextDependencyLoading() {
// try to import library does not exist on classpath. it'll fail
assertEquals(InterpreterResult.Code.ERROR, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
// load library from maven repository and try to import again
repl.interpret("z.load(\"org.apache.commons:commons-csv:1.1\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
}
@Test
public void emptyConfigurationVariablesOnlyForNonSparkProperties() {
Properties intpProperty = repl.getProperty();

5
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -24,6 +24,7 @@ import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
@ -64,8 +65,10 @@ public class SparkSqlInterpreterTest {
sql.setInterpreterGroup(intpGroup);
sql.open();
}
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {

70
testing/downloadSpark.sh Executable file
View file

@ -0,0 +1,70 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
if [ $# -ne 2 ]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
fi
else
SPARK_VER_RANGE=">1.3"
fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
echo "SPARK_HOME is ${SPARK_HOME}"
if [ ! -d "${SPARK_HOME}" ]; then
if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
STARTTIME=`date +%s`
timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
STARTTIME=`date +%s`
timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}"
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
set +xe

40
testing/startSparkCluster.sh
View file

@ -26,41 +26,41 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
fi
else
SPARK_VER_RANGE=">1.3"
fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
echo "SPARK_HOME is ${SPARK_HOME} "
if [ ! -d "${SPARK_HOME}" ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
wget -q "${PREFFERED}${PATHINFO}"
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
echo "SPARK_HOME is ${SPARK_HOME}"
# create PID dir. test case detect pid file so they can select active spark home dir for test
mkdir -p ${SPARK_HOME}/run
export SPARK_PID_DIR=${SPARK_HOME}/run
# start
export SPARK_MASTER_PORT=7071
export SPARK_MASTER_WEBUI_PORT=7072
export SPARK_WORKER_WEBUI_PORT=8082
${SPARK_HOME}/sbin/start-master.sh
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then # spark 1.3 or prior
if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
# spark 1.3 or prior
${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
else
${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
fi
set +xe

4
testing/stopSparkCluster.sh
View file

@ -25,6 +25,8 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
@ -32,6 +34,6 @@ export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VER
# set create PID dir
export SPARK_PID_DIR=${SPARK_HOME}/run
${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
${SPARK_HOME}/sbin/stop-master.sh
set +xe

187
zeppelin-display/pom.xml Normal file
View file

@ -0,0 +1,187 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-display</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Display system apis</name>
<url>http://incubator.zeppelin.apache.org</url>
<properties>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scalap</artifactId>
<version>${scala.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.1.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.16</version>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
<configuration>
<argLine>-Xmx2048m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

192
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala Normal file
View file

@ -0,0 +1,192 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular
import java.io.PrintStream
import org.apache.zeppelin.display.{AngularObjectWatcher, AngularObject}
import org.apache.zeppelin.interpreter.{InterpreterResult, InterpreterContext}
import scala.xml._
/**
* Element that binded to Angular object
*/
abstract class AbstractAngularElem(val interpreterContext: InterpreterContext,
val modelName: String,
val angularObjects: Map[String, AngularObject[Any]],
prefix: String,
label: String,
attributes1: MetaData,
scope: NamespaceBinding,
minimizeEmpty: Boolean,
child: Node*)
extends Elem(prefix, label, attributes1, scope, minimizeEmpty, child:_*) {
val uniqueId = java.util.UUID.randomUUID.toString.replaceAll("-", "_")
/**
* On click element
* @param callback
* @return
*/
def onClick(callback: () => Unit): AbstractAngularElem = {
onEvent("ng-click", callback)
}
/**
* On
* @param callback
* @return
*/
def onChange(callback: () => Unit): AbstractAngularElem = {
onEvent("ng-change", callback)
}
/**
* Bind angularObject to ng-model directive
* @param name name of angularObject
* @param value initialValue
* @return
*/
def model(name: String, value: Any): AbstractAngularElem = {
val registry = interpreterContext.getAngularObjectRegistry
// create AngularFunction in current paragraph
val elem = this % Attribute(None, "ng-model",
Text(s"${name}"),
Null)
val angularObject = addAngularObject(name, value)
.asInstanceOf[AngularObject[Any]]
newElem(
interpreterContext,
name,
angularObjects + (name -> angularObject),
elem)
}
def model(name: String): AbstractAngularElem = {
val registry = interpreterContext.getAngularObjectRegistry
// create AngularFunction in current paragraph
val elem = this % Attribute(None, "ng-model",
Text(s"${name}"),
Null)
newElem(
interpreterContext,
name,
angularObjects,
elem)
}
/**
* Retrieve value of model
* @return
*/
def model(): Any = {
if (angularObjects.contains(modelName)) {
angularObjects(modelName).get()
} else {
None
}
}
/**
*
* @param eventName angular directive like ng-click, ng-change, etc.
* @return
*/
def onEvent(eventName: String, callback: () => Unit): AbstractAngularElem = {
val registry = interpreterContext.getAngularObjectRegistry
// create AngularFunction in current paragraph
val functionName = eventName.replaceAll("-", "_") + "_" + uniqueId
val elem = this % Attribute(None, eventName,
Text(s"${functionName}=$$event.timeStamp"),
Null)
val angularObject = addAngularObject(functionName, "")
angularObject.addWatcher(new AngularObjectWatcher(interpreterContext) {
override def watch(oldObject: scala.Any, newObject: scala.Any, context: InterpreterContext)
:Unit = {
InterpreterContext.set(interpreterContext)
callback()
}
})
newElem(
interpreterContext,
modelName,
angularObjects + (eventName -> angularObject),
elem)
}
protected def addAngularObject(name: String, value: Any): AngularObject[Any]
protected def newElem(interpreterContext: InterpreterContext,
name: String,
angularObjects: Map[String, AngularObject[Any]],
elem: scala.xml.Elem): AbstractAngularElem
/**
* disassociate this element and it's child from front-end
* by removing angularobject
*/
def disassociate() = {
remove(this)
}
/**
* Remove all angularObject recursively
* @param node
*/
private def remove(node: Node): Unit = {
if (node.isInstanceOf[AbstractAngularElem]) {
node.asInstanceOf[AbstractAngularElem].angularObjects.values.foreach{ ao =>
interpreterContext.getAngularObjectRegistry.remove(ao.getName, ao.getNoteId, ao
.getParagraphId)
}
}
node.child.foreach(remove _)
}
/**
* Print into provided print stream
* @return
*/
def display(out: java.io.PrintStream): Unit = {
out.print(this.toString)
out.flush()
}
/**
* Print into InterpreterOutput
*/
def display(): Unit = {
val out = interpreterContext.out
out.setType(InterpreterResult.Type.ANGULAR)
out.write(this.toString())
out.flush()
}
}

95
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularModel.scala Normal file
View file

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular
import org.apache.zeppelin.display.AngularObject
import org.apache.zeppelin.interpreter.InterpreterContext
/**
* Represents ng-model with angular object
*/
abstract class AbstractAngularModel(name: String) {
val context = InterpreterContext.get
val registry = context.getAngularObjectRegistry
/**
* Create AngularModel with initial Value
* @param name name of model
* @param newValue value
*/
def this(name: String, newValue: Any) = {
this(name)
value(newValue)
}
protected def getAngularObject(): AngularObject[Any]
protected def addAngularObject(value: Any): AngularObject[Any]
/**
* Get value of the model
* @return
*/
def apply(): Any = {
value()
}
/**
* Get value of the model
* @return
*/
def value(): Any = {
val angularObject = getAngularObject()
if (angularObject == null) {
None
} else {
angularObject.get
}
}
def apply(newValue: Any): Unit = {
value(newValue)
}
/**
* Set value of the model
* @param newValue
*/
def value(newValue: Any): Unit = {
var angularObject = getAngularObject()
if (angularObject == null) {
// create new object
angularObject = addAngularObject(newValue)
} else {
angularObject.set(newValue)
}
angularObject.get()
}
def remove(): Any = {
val angularObject = getAngularObject()
if (angularObject == null) {
None
} else {
registry.remove(name, angularObject.getNoteId(), angularObject.getParagraphId())
angularObject.get
}
}
}

84
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElem.scala Normal file
View file

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.notebookscope
import org.apache.zeppelin.display.angular.AbstractAngularElem
import org.apache.zeppelin.display.{angular, AngularObject}
import org.apache.zeppelin.interpreter.InterpreterContext
import scala.collection.JavaConversions
import scala.xml._
/**
* AngularElement in notebook scope
*/
class AngularElem(override val interpreterContext: InterpreterContext,
override val modelName: String,
override val angularObjects: Map[String, AngularObject[Any]],
prefix: String,
label: String,
attributes1: MetaData,
scope: NamespaceBinding,
minimizeEmpty: Boolean,
child: Node*)
extends AbstractAngularElem(
interpreterContext, modelName, angularObjects, prefix, label, attributes1, scope,
minimizeEmpty, child: _*) {
override protected def addAngularObject(name: String, value: Any): AngularObject[Any] = {
val registry = interpreterContext.getAngularObjectRegistry
registry.add(name, value, interpreterContext.getNoteId, null).asInstanceOf[AngularObject[Any]]
}
override protected def newElem(interpreterContext: InterpreterContext,
name: String,
angularObjects: Map[String, AngularObject[Any]],
elem: scala.xml.Elem): angular.AbstractAngularElem = {
new AngularElem(
interpreterContext,
name,
angularObjects,
elem.prefix,
elem.label,
elem.attributes,
elem.scope,
elem.minimizeEmpty,
elem.child:_*)
}
}
object AngularElem {
implicit def Elem2AngularDisplayElem(elem: Elem): AbstractAngularElem = {
new AngularElem(InterpreterContext.get(), null,
Map[String, AngularObject[Any]](),
elem.prefix, elem.label, elem.attributes, elem.scope, elem.minimizeEmpty, elem.child:_*);
}
/**
* Disassociate (remove) all angular object in this notebook
*/
def disassociate() = {
val ic = InterpreterContext.get
val registry = ic.getAngularObjectRegistry
JavaConversions.asScalaBuffer(registry.getAll(ic.getNoteId, null)).foreach(ao =>
registry.remove(ao.getName, ao.getNoteId, null)
)
}
}

52
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/notebookscope/AngularModel.scala Normal file
View file

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.notebookscope
import org.apache.zeppelin.display.AngularObject
import org.apache.zeppelin.display.angular.AbstractAngularModel
import org.apache.zeppelin.interpreter.InterpreterContext
/**
* Represents ng-model in notebook scope
*/
class AngularModel(name: String)
extends org.apache.zeppelin.display.angular.AbstractAngularModel(name) {
def this(name: String, newValue: Any) = {
this(name)
value(newValue)
}
override protected def getAngularObject(): AngularObject[Any] = {
registry.get(name, context.getNoteId, null).asInstanceOf[AngularObject[Any]]
}
override protected def addAngularObject(value: Any): AngularObject[Any] = {
registry.add(name, value, context.getNoteId, null).asInstanceOf[AngularObject[Any]]
}
}
object AngularModel {
def apply(name: String): AbstractAngularModel = {
new AngularModel(name)
}
def apply(name: String, newValue: Any): AbstractAngularModel = {
new AngularModel(name, newValue)
}
}

86
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElem.scala Normal file
View file

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.paragraphscope
import org.apache.zeppelin.display.angular.AbstractAngularElem
import org.apache.zeppelin.display.{angular, AngularObject}
import org.apache.zeppelin.interpreter.InterpreterContext
import scala.collection.JavaConversions
import scala.xml._
/**
* AngularElement in paragraph scope
*/
class AngularElem(override val interpreterContext: InterpreterContext,
override val modelName: String,
override val angularObjects: Map[String, AngularObject[Any]],
prefix: String,
label: String,
attributes1: MetaData,
scope: NamespaceBinding,
minimizeEmpty: Boolean,
child: Node*)
extends AbstractAngularElem(
interpreterContext, modelName, angularObjects, prefix, label, attributes1, scope,
minimizeEmpty, child: _*) {
override protected def addAngularObject(name: String, value: Any): AngularObject[Any] = {
val registry = interpreterContext.getAngularObjectRegistry
registry.add(name, value, interpreterContext.getNoteId, interpreterContext.getParagraphId)
.asInstanceOf[AngularObject[Any]]
}
override protected def newElem(interpreterContext: InterpreterContext,
name: String,
angularObjects: Map[String, AngularObject[Any]],
elem: scala.xml.Elem): angular.AbstractAngularElem = {
new AngularElem(
interpreterContext,
name,
angularObjects,
elem.prefix,
elem.label,
elem.attributes,
elem.scope,
elem.minimizeEmpty,
elem.child:_*)
}
}
object AngularElem {
implicit def Elem2AngularDisplayElem(elem: Elem): AbstractAngularElem = {
new AngularElem(InterpreterContext.get(), null,
Map[String, AngularObject[Any]](),
elem.prefix, elem.label, elem.attributes, elem.scope, elem.minimizeEmpty, elem.child:_*);
}
/**
* Disassociate (remove) all angular object in this notebook
*/
def disassociate() = {
val ic = InterpreterContext.get
val registry = ic.getAngularObjectRegistry
JavaConversions.asScalaBuffer(registry.getAll(ic.getNoteId, ic.getParagraphId)).foreach(ao =>
registry.remove(ao.getName, ao.getNoteId, ao.getParagraphId)
)
}
}

53
zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularModel.scala Normal file
View file

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.paragraphscope
import org.apache.zeppelin.display.AngularObject
import org.apache.zeppelin.display.angular.AbstractAngularModel
/**
* Represents ng-model in paragraph scope
*/
class AngularModel(name: String)
extends org.apache.zeppelin.display.angular.AbstractAngularModel(name) {
def this(name: String, newValue: Any) = {
this(name)
value(newValue)
}
override protected def getAngularObject(): AngularObject[Any] = {
registry.get(name,
context.getNoteId, context.getParagraphId).asInstanceOf[AngularObject[Any]]
}
override protected def addAngularObject(value: Any): AngularObject[Any] = {
registry.add(name, value,
context.getNoteId, context.getParagraphId).asInstanceOf[AngularObject[Any]]
}
}
object AngularModel {
def apply(name: String): AbstractAngularModel = {
new AngularModel(name)
}
def apply(name: String, newValue: Any): AbstractAngularModel = {
new AngularModel(name, newValue)
}
}

156
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularElemTest.scala Normal file
View file

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular
import java.io.{ByteArrayOutputStream, PrintStream}
import java.util
import org.apache.zeppelin.display.{AngularObject, AngularObjectRegistry, GUI}
import org.apache.zeppelin.interpreter._
import org.apache.zeppelin.user.AuthenticationInfo
import org.scalatest.concurrent.Eventually
import org.scalatest.{BeforeAndAfter, BeforeAndAfterEach, FlatSpec, Matchers}
/**
* Test
*/
trait AbstractAngularElemTest
extends FlatSpec with BeforeAndAfter with BeforeAndAfterEach with Eventually with Matchers {
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "paragraph", "title", "text",
new AuthenticationInfo(), new util.HashMap[String, Object](), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new util.LinkedList[InterpreterContextRunner](),
new InterpreterOutput(new InterpreterOutputListener() {
override def onAppend(out: InterpreterOutput, line: Array[Byte]): Unit = {
// nothing to do
}
override def onUpdate(out: InterpreterOutput, output: Array[Byte]): Unit = {
// nothing to do
}
}))
InterpreterContext.set(context)
super.beforeEach() // To be stackable, must call super.beforeEach
}
def angularElem(elem: scala.xml.Elem): AbstractAngularElem;
def angularModel(name: String): AbstractAngularModel;
"AngularElem" should "provide onclick method" in {
registrySize should be(0)
var a = 0
val elem = angularElem(<div></div>).onClick(() => {
a = a + 1
})
elem.angularObjects.get("ng-click") should not be(null)
registrySize should be(1)
// click create thread for callback function to run. So it'll may not immediately invoked
// after click. therefore eventually should be
click(elem)
eventually {
a should be(1)
}
click(elem)
eventually {
a should be(2)
}
// disassociate
elem.disassociate()
registrySize should be(0)
}
"AngularElem" should "print angular display directive only once in a paragraph" in {
val out = new ByteArrayOutputStream()
val printOut = new PrintStream(out)
angularElem(<div></div>).display(printOut)
out.toString should be("<div></div>")
out.reset
angularElem(<div></div>).display(printOut)
out.toString should be("<div></div>")
}
"AngularElem" should "bind angularObject to ng-model directive " in {
angularElem(<div></div>)
.model("name", "value").toString should be("<div ng-model=\"name\"></div>")
angularElem(<div></div>).model("name", "value").model() should be("value")
angularElem(<div></div>).model() should be(None)
}
"AngularElem" should "able to disassociate AngularObjects" in {
val elem1 = angularElem(<div></div>).model("name1", "value1")
val elem2 = angularElem(<div></div>).model("name2", "value2")
val elem3 = angularElem(<div></div>).model("name3", "value3")
registrySize should be(3)
elem1.disassociate()
registrySize should be(2)
elem2.disassociate()
elem3.disassociate()
registrySize should be(0)
}
"AngularElem" should "allow access to InterpreterContext inside of callback function" in {
angularModel("name").value("value")
var modelValue = ""
val elem = angularElem(<div></div>).onClick(() =>
modelValue = angularModel("name")().toString
)
click(elem)
eventually { modelValue should be("value")}
}
def registry = {
InterpreterContext.get().getAngularObjectRegistry
}
def registrySize = {
registry.getAllWithGlobal("note").size()
}
def noteId = {
InterpreterContext.get().getNoteId
}
def click(elem: org.apache.zeppelin.display.angular.AbstractAngularElem) = {
fireEvent("ng-click", elem)
}
// simulate click
def fireEvent(eventName: String, elem: org.apache.zeppelin.display.angular.AbstractAngularElem) = {
val angularObject: AngularObject[Any] = elem.angularObjects(eventName);
angularObject.set("event");
}
}

99
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularModelTest.scala Normal file
View file

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular
import org.apache.zeppelin.display.{AngularObjectRegistry, GUI}
import org.apache.zeppelin.interpreter._
import org.apache.zeppelin.user.AuthenticationInfo
import org.scalatest.concurrent.Eventually
import org.scalatest.{BeforeAndAfter, BeforeAndAfterEach, FlatSpec, Matchers}
/**
* Abstract Test for AngularModel
*/
trait AbstractAngularModelTest extends FlatSpec
with BeforeAndAfter with BeforeAndAfterEach with Eventually with Matchers {
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new java.util.HashMap[String, Object](), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
null,
new java.util.LinkedList[InterpreterContextRunner](),
new InterpreterOutput(new InterpreterOutputListener() {
override def onAppend(out: InterpreterOutput, line: Array[Byte]): Unit = {
// nothing to do
}
override def onUpdate(out: InterpreterOutput, output: Array[Byte]): Unit = {
// nothing to do
}
}))
InterpreterContext.set(context)
super.beforeEach() // To be stackable, must call super.beforeEach
}
def angularModel(name: String): AbstractAngularModel
def angularModel(name: String, value: Any): AbstractAngularModel
"AngularModel" should "able to create AngularObject" in {
val registry = InterpreterContext.get().getAngularObjectRegistry
registrySize should be(0)
angularModel("model1")() should be(None)
registrySize should be(0)
angularModel("model1", "value1")() should be("value1")
registrySize should be(1)
angularModel("model1")() should be("value1")
registrySize should be(1)
}
"AngularModel" should "able to update AngularObject" in {
val registry = InterpreterContext.get().getAngularObjectRegistry
val model1 = angularModel("model1", "value1")
model1() should be("value1")
registrySize should be(1)
model1.value("newValue1")
model1() should be("newValue1")
registrySize should be(1)
angularModel("model1", "value2")() should be("value2")
registrySize should be(1)
}
"AngularModel" should "able to remove AngularObject" in {
angularModel("model1", "value1")
registrySize should be(1)
angularModel("model1").remove()
registrySize should be(0)
}
def registry() = {
InterpreterContext.get().getAngularObjectRegistry
}
def registrySize() = {
registry().getAllWithGlobal(InterpreterContext.get().getNoteId).size
}
}

41
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/notebookscope/AngularElemTest.scala Normal file
View file

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.notebookscope
import org.apache.zeppelin.display.angular.{AbstractAngularElem, AbstractAngularModel, AbstractAngularElemTest}
import scala.xml.Elem
/**
* Test
*/
class AngularElemTest extends AbstractAngularElemTest {
override def angularElem(elem: Elem): AbstractAngularElem = {
AngularElem.Elem2AngularDisplayElem(elem)
}
override def angularModel(name: String): AbstractAngularModel = {
AngularModel(name)
}
"AngularElem" should "able to be created from implicit conversion" in {
import AngularElem._
<div></div>.model("modelname")
}
}

32
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/notebookscope/AngularModelTest.scala Normal file
View file

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.notebookscope
import org.apache.zeppelin.display.angular.{AbstractAngularModel, AbstractAngularModelTest}
/**
* Test for AngularModel
*/
class AngularModelTest extends AbstractAngularModelTest {
override def angularModel(name: String): AbstractAngularModel = {
AngularModel(name)
}
override def angularModel(name: String, value: Any): AbstractAngularModel = {
AngularModel(name, value)
}
}

41
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularElemTest.scala Normal file
View file

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.paragraphscope
import org.apache.zeppelin.display.angular.{AbstractAngularElem, AbstractAngularModel, AbstractAngularElemTest}
import scala.xml.Elem
/**
* Test
*/
class AngularElemTest extends AbstractAngularElemTest {
override def angularElem(elem: Elem): AbstractAngularElem = {
AngularElem.Elem2AngularDisplayElem(elem)
}
override def angularModel(name: String): AbstractAngularModel = {
AngularModel(name)
}
"AngularElem" should "able to be created from implicit conversion" in {
import AngularElem._
<div></div>.model("modelname")
}
}

32
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/paragraphscope/AngularModelTest.scala Normal file
View file

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.display.angular.paragraphscope
import org.apache.zeppelin.display.angular.{AbstractAngularModel, AbstractAngularModelTest}
/**
* Test for AngularModel
*/
class AngularModelTest extends AbstractAngularModelTest {
override def angularModel(name: String): AbstractAngularModel = {
AngularModel(name)
}
override def angularModel(name: String, value: Any): AbstractAngularModel = {
AngularModel(name, value)
}
}

6
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -37,6 +37,7 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Cassandra (http://cassandra.apache.org/)
(Apache 2.0) Apache CXF (http://cxf.apache.org/)
(Apache 2.0) Apache Hive (http://hive.apache.org/)
(Apache 2.0) Apache HBase (http://hbase.apache.org/)
(Apache 2.0) Apache Ignite (http://ignite.apache.org/)
(Apache 2.0) Apache Kylin (http://kylin.apache.org/)
(Apache 2.0) Apache Lens (http://lens.apache.org/)
@ -94,10 +95,12 @@ The following components are provided under Apache License.
(Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org)
(Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org)
(Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org)
(Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.4.1 - https://github.com/google/protobuf/releases)
(Apache 2.0) Tachyon Shell (org.tachyonproject:tachyon-shell:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Servers (org.tachyonproject:tachyon-servers:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Minicluster (org.tachyonproject:tachyon-minicluster:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Underfs Local (org.tachyonproject:tachyon-underfs-local:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Microsoft Azure Storage Library for Java (com.microsoft.azure:azure-storage:4.0.0 - https://github.com/Azure/azure-storage-java)
@ -154,6 +157,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(BSD Style) dom4j v1.6.1 (http://www.dom4j.org) - https://github.com/dom4j/dom4j/blob/dom4j_1_6_1/LICENSE.txt
(BSD Style) JSch v0.1.53 (http://www.jcraft.com) - http://www.jcraft.com/jsch/LICENSE.txt
(BSD 3 Clause) highlightjs v8.4.0 (https://highlightjs.org/) - https://github.com/isagalaev/highlight.js/blob/8.4/LICENSE
(BSD 3 Clause) hamcrest v1.3 (http://hamcrest.org/JavaHamcrest/) - http://opensource.org/licenses/BSD-3-Clause
(BSD Style) JLine v2.12.1 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt
@ -194,6 +199,7 @@ The following components are provided under the EPL License.
(EPL 1.0) Aether (org.sonatype.aether - http://www.eclipse.org/aether/)
(EPL 1.0) JDT Annotations For Enhanced Null Analysis (org.eclipse.jdt:org.eclipse.jdt.annotation:1.1.0 - https://repo.eclipse.org/content/repositories/eclipse-releases/org/eclipse/jdt/org.eclipse.jdt.annotation)
(EPL 1.0) JRuby (org.jruby.jruby-complete:v1.6.8 - http://www.jruby.org/)
========================================================================

202
zeppelin-distribution/src/bin_license/licenses/LICENSE-azure-storage-4.0.0 Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

26
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
View file

@ -24,7 +24,9 @@ import java.util.List;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.RepositoryPolicy;
import org.sonatype.aether.resolution.ArtifactResult;
/**
@ -41,12 +43,32 @@ public abstract class AbstractDependencyResolver {
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
}
public List<RemoteRepository> getRepos() {
return this.repos;
}
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(snapshot, null);
rr.setPolicy(true, new RepositoryPolicy(
snapshot,
RepositoryPolicy.UPDATE_POLICY_DAILY,
RepositoryPolicy.CHECKSUM_POLICY_WARN));
repos.add(rr);
}
}
public void addRepo(String id, String url, boolean snapshot, Authentication auth) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(true, new RepositoryPolicy(
snapshot,
RepositoryPolicy.UPDATE_POLICY_DAILY,
RepositoryPolicy.CHECKSUM_POLICY_WARN));
rr.setAuthentication(auth);
repos.add(rr);
}
}
@ -54,7 +76,7 @@ public abstract class AbstractDependencyResolver {
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
if (it.hasNext()) {
while (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
View file

@ -127,7 +127,7 @@ public class DependencyContext {
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
collectRequest.addRepository(rr);
}

57
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.dep;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
@ -28,6 +29,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
@ -56,19 +58,21 @@ public class DependencyResolver extends AbstractDependencyResolver {
super(localRepoPath);
}
public List<File> load(String artifact) throws Exception {
public List<File> load(String artifact)
throws RepositoryException, IOException {
return load(artifact, new LinkedList<String>());
}
public List<File> load(String artifact, String destPath) throws Exception {
public List<File> load(String artifact, String destPath)
throws RepositoryException, IOException {
return load(artifact, new LinkedList<String>(), destPath);
}
public synchronized List<File> load(String artifact, Collection<String> excludes)
throws Exception {
throws RepositoryException, IOException {
if (StringUtils.isBlank(artifact)) {
// Should throw here
throw new RuntimeException("Invalid artifact to load");
// Skip dependency loading if artifact is empty
return new LinkedList<File>();
}
// <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>
@ -83,29 +87,34 @@ public class DependencyResolver extends AbstractDependencyResolver {
}
public List<File> load(String artifact, Collection<String> excludes, String destPath)
throws Exception {
List<File> libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile: libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
throws RepositoryException, IOException {
List<File> libs = new LinkedList<File>();
if (StringUtils.isNotBlank(artifact)) {
libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile : libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}
}
return libs;
}
private List<File> loadFromMvn(String artifact, Collection<String> excludes) throws Exception {
private List<File> loadFromMvn(String artifact, Collection<String> excludes)
throws RepositoryException {
Collection<String> allExclusions = new LinkedList<String>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
@ -142,7 +151,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception {
Collection<String> excludes) throws RepositoryException {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
View file

@ -23,13 +23,13 @@ import org.sonatype.aether.repository.Authentication;
*/
public class Repository {
private boolean snapshot = false;
private String name;
private String id;
private String url;
private String username = null;
private String password = null;
public Repository(String name){
this.name = name;
public Repository(String id){
this.id = id;
}
public Repository url(String url) {
@ -46,8 +46,8 @@ public class Repository {
return snapshot;
}
public String getName() {
return name;
public String getId() {
return id;
}
public String getUrl() {

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -127,10 +127,6 @@ public abstract class Interpreter {
}
}
public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
private InterpreterGroup interpreterGroup;
private URL [] classloaderUrls;

16
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -21,7 +21,9 @@ import java.util.List;
import java.util.Map;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.resource.ResourcePool;
/**
* Interpreter context
@ -47,18 +49,22 @@ public class InterpreterContext {
private final String paragraphTitle;
private final String paragraphId;
private final String paragraphText;
private AuthenticationInfo authenticationInfo;
private final Map<String, Object> config;
private GUI gui;
private AngularObjectRegistry angularObjectRegistry;
private ResourcePool resourcePool;
private List<InterpreterContextRunner> runners;
public InterpreterContext(String noteId,
String paragraphId,
String paragraphTitle,
String paragraphText,
AuthenticationInfo authenticationInfo,
Map<String, Object> config,
GUI gui,
AngularObjectRegistry angularObjectRegistry,
ResourcePool resourcePool,
List<InterpreterContextRunner> runners,
InterpreterOutput out
) {
@ -66,9 +72,11 @@ public class InterpreterContext {
this.paragraphId = paragraphId;
this.paragraphTitle = paragraphTitle;
this.paragraphText = paragraphText;
this.authenticationInfo = authenticationInfo;
this.config = config;
this.gui = gui;
this.angularObjectRegistry = angularObjectRegistry;
this.resourcePool = resourcePool;
this.runners = runners;
this.out = out;
}
@ -90,6 +98,10 @@ public class InterpreterContext {
return paragraphTitle;
}
public AuthenticationInfo getAuthenticationInfo() {
return authenticationInfo;
}
public Map<String, Object> getConfig() {
return config;
}
@ -102,6 +114,10 @@ public class InterpreterContext {
return angularObjectRegistry;
}
public ResourcePool getResourcePool() {
return resourcePool;
}
public List<InterpreterContextRunner> getRunners() {
return runners;
}

31
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -17,14 +17,13 @@
package org.apache.zeppelin.interpreter;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
/**
* InterpreterGroup is list of interpreters in the same group.
@ -37,13 +36,27 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
AngularObjectRegistry angularObjectRegistry;
RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
ResourcePool resourcePool;
private static final Map<String, InterpreterGroup> allInterpreterGroups =
new ConcurrentHashMap<String, InterpreterGroup>();
public static InterpreterGroup get(String id) {
return allInterpreterGroups.get(id);
}
public static Collection<InterpreterGroup> getAll() {
return new LinkedList(allInterpreterGroups.values());
}
public InterpreterGroup(String id) {
this.id = id;
allInterpreterGroups.put(id, this);
}
public InterpreterGroup() {
getId();
allInterpreterGroups.put(id, this);
}
private static String generateId() {
@ -135,5 +148,15 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
remoteInterpreterProcess.dereference();
}
}
allInterpreterGroups.remove(id);
}
public void setResourcePool(ResourcePool resourcePool) {
this.resourcePool = resourcePool;
}
public ResourcePool getResourcePool() {
return resourcePool;
}
}

49
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -30,9 +30,7 @@ import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
@ -53,6 +51,7 @@ public class RemoteInterpreter extends Interpreter {
Gson gson = new Gson();
private String interpreterRunner;
private String interpreterPath;
private String localRepoPath;
private String className;
FormType formType;
boolean initialized;
@ -61,17 +60,19 @@ public class RemoteInterpreter extends Interpreter {
private int maxPoolSize;
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
int connectTimeout,
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
String className,
String interpreterRunner,
String interpreterPath,
String localRepoPath,
int connectTimeout,
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
env = new HashMap<String, String>();
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
@ -79,16 +80,18 @@ public class RemoteInterpreter extends Interpreter {
}
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
String className,
String interpreterRunner,
String interpreterPath,
String localRepoPath,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
this.env = env;
this.connectTimeout = connectTimeout;
this.maxPoolSize = 10;
@ -110,8 +113,8 @@ public class RemoteInterpreter extends Interpreter {
if (intpGroup.getRemoteInterpreterProcess() == null) {
// create new remote process
RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess(
interpreterRunner, interpreterPath, env, connectTimeout,
remoteInterpreterProcessListener);
interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
remoteInterpreterProcessListener);
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}
@ -143,8 +146,9 @@ public class RemoteInterpreter extends Interpreter {
try {
for (Interpreter intp : this.getInterpreterGroup()) {
logger.info("Create remote interpreter {}", intp.getClassName());
client.createInterpreter(intp.getClassName(), (Map) property);
property.put("zeppelin.interpreter.localRepo", localRepoPath);
client.createInterpreter(getInterpreterGroup().getId(),
intp.getClassName(), (Map) property);
}
} catch (TException e) {
broken = true;
@ -172,7 +176,9 @@ public class RemoteInterpreter extends Interpreter {
boolean broken = false;
try {
client = interpreterProcess.getClient();
client.close(className);
if (client != null) {
client.close(className);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
@ -291,6 +297,10 @@ public class RemoteInterpreter extends Interpreter {
@Override
public int getProgress(InterpreterContext context) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null || !interpreterProcess.isRunning()) {
return 0;
}
Client client = null;
try {
client = interpreterProcess.getClient();
@ -354,6 +364,7 @@ public class RemoteInterpreter extends Interpreter {
ic.getParagraphId(),
ic.getParagraphTitle(),
ic.getParagraphText(),
gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()),
gson.toJson(ic.getGui()),
gson.toJson(ic.getRunners()));

245
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java Normal file
View file

@ -0,0 +1,245 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import com.google.gson.Gson;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
import org.apache.zeppelin.resource.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* Thread connection ZeppelinServer -> RemoteInterpreterServer does not provide
* remote method invocation from RemoteInterpreterServer -> ZeppelinServer
*
* This class provides event send and get response from RemoteInterpreterServer to
* ZeppelinServer.
*
* RemoteInterpreterEventPoller is counter part in ZeppelinServer
*/
public class RemoteInterpreterEventClient implements ResourcePoolConnector {
private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEvent.class);
private final List<RemoteInterpreterEvent> eventQueue = new LinkedList<RemoteInterpreterEvent>();
private final List<ResourceSet> getAllResourceResponse = new LinkedList<ResourceSet>();
private final Map<ResourceId, Object> getResourceResponse = new HashMap<ResourceId, Object>();
private final Gson gson = new Gson();
/**
* Run paragraph
* @param runner
*/
public void run(InterpreterContextRunner runner) {
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER,
gson.toJson(runner)));
}
/**
* notify new angularObject creation
* @param object
*/
public void angularObjectAdd(AngularObject object) {
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.ANGULAR_OBJECT_ADD, gson.toJson(object)));
}
/**
* notify angularObject update
*/
public void angularObjectUpdate(AngularObject object) {
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE, gson.toJson(object)));
}
/**
* notify angularObject removal
*/
public void angularObjectRemove(String name, String noteId, String paragraphId) {
Map<String, String> removeObject = new HashMap<String, String>();
removeObject.put("name", name);
removeObject.put("noteId", noteId);
removeObject.put("paragraphId", paragraphId);
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE, gson.toJson(removeObject)));
}
/**
* Get all resources except for specific resourcePool
* @return
*/
@Override
public ResourceSet getAllResources() {
// request
sendEvent(new RemoteInterpreterEvent(RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL, null));
synchronized (getAllResourceResponse) {
while (getAllResourceResponse.isEmpty()) {
try {
getAllResourceResponse.wait();
} catch (InterruptedException e) {
logger.warn(e.getMessage(), e);
}
}
ResourceSet resourceSet = getAllResourceResponse.remove(0);
return resourceSet;
}
}
@Override
public Object readResource(ResourceId resourceId) {
logger.debug("Request Read Resource {} from ZeppelinServer", resourceId.getName());
synchronized (getResourceResponse) {
// wait for previous response consumed
while (getResourceResponse.containsKey(resourceId)) {
try {
getResourceResponse.wait();
} catch (InterruptedException e) {
logger.warn(e.getMessage(), e);
}
}
// send request
Gson gson = new Gson();
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.RESOURCE_GET,
gson.toJson(resourceId)));
// wait for response
while (!getResourceResponse.containsKey(resourceId)) {
try {
getResourceResponse.wait();
} catch (InterruptedException e) {
logger.warn(e.getMessage(), e);
}
}
Object o = getResourceResponse.remove(resourceId);
getResourceResponse.notifyAll();
return o;
}
}
/**
* Supposed to call from RemoteInterpreterEventPoller
*/
public void putResponseGetAllResources(List<String> resources) {
logger.debug("ResourceSet from ZeppelinServer");
ResourceSet resourceSet = new ResourceSet();
for (String res : resources) {
RemoteResource resource = gson.fromJson(res, RemoteResource.class);
resource.setResourcePoolConnector(this);
resourceSet.add(resource);
}
synchronized (getAllResourceResponse) {
getAllResourceResponse.add(resourceSet);
getAllResourceResponse.notify();
}
}
/**
* Supposed to call from RemoteInterpreterEventPoller
* @param resourceId json serialized ResourceId
* @param object java serialized of the object
*/
public void putResponseGetResource(String resourceId, ByteBuffer object) {
ResourceId rid = gson.fromJson(resourceId, ResourceId.class);
logger.debug("Response resource {} from RemoteInterpreter", rid.getName());
Object o = null;
try {
o = Resource.deserializeObject(object);
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (ClassNotFoundException e) {
logger.error(e.getMessage(), e);
}
synchronized (getResourceResponse) {
getResourceResponse.put(rid, o);
getResourceResponse.notifyAll();
}
}
/**
* Supposed to call from RemoteInterpreterEventPoller
* @return next available event
*/
public RemoteInterpreterEvent pollEvent() {
synchronized (eventQueue) {
if (eventQueue.isEmpty()) {
try {
eventQueue.wait(1000);
} catch (InterruptedException e) {
}
}
if (eventQueue.isEmpty()) {
return new RemoteInterpreterEvent(RemoteInterpreterEventType.NO_OP, "");
} else {
RemoteInterpreterEvent event = eventQueue.remove(0);
logger.debug("Send event {}", event.getType());
return event;
}
}
}
public void onInterpreterOutputAppend(String noteId, String paragraphId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", output);
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_APPEND,
gson.toJson(appendOutput)));
}
public void onInterpreterOutputUpdate(String noteId, String paragraphId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", output);
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_UPDATE,
gson.toJson(appendOutput)));
}
private void sendEvent(RemoteInterpreterEvent event) {
synchronized (eventQueue) {
eventQueue.add(event);
eventQueue.notifyAll();
}
}
}

133
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -28,13 +28,21 @@ import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourceId;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
*
* Processes message from RemoteInterpreter process
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
@ -117,6 +125,15 @@ public class RemoteInterpreterEventPoller extends Thread {
interpreterProcess.getInterpreterContextRunnerPool().run(
runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) {
ResourceSet resourceSet = getAllResourcePoolExcept();
sendResourcePoolResponseGetAll(resourceSet);
} else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) {
String resourceIdString = event.getData();
ResourceId resourceId = gson.fromJson(resourceIdString, ResourceId.class);
logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName());
Object o = getResource(resourceId);
sendResourceResponseGet(resourceId, o);
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
// on output append
Map<String, String> outputAppend = gson.fromJson(
@ -143,6 +160,120 @@ public class RemoteInterpreterEventPoller extends Thread {
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
List<String> resourceList = new LinkedList<String>();
Gson gson = new Gson();
for (Resource r : resourceSet) {
resourceList.add(gson.toJson(r));
}
client.resourcePoolResponseGetAll(resourceList);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
}
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resoucePoolGetAll();
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(gson.fromJson(res, Resource.class));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
return resourceSet;
}
private void sendResourceResponseGet(ResourceId resourceId, Object o) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
Gson gson = new Gson();
String rid = gson.toJson(resourceId);
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseGet(rid, obj);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
}
private Object getResource(ResourceId resourceId) {
InterpreterGroup intpGroup = InterpreterGroup.get(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
return localPool.get(resourceId.getName());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
ByteBuffer res = client.resourceGet(resourceId.getName());
Object o = Resource.deserializeObject(res);
return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
return null;
}
private void waitQuietly() {
try {
synchronized (this) {

35
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -45,6 +45,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int port = -1;
private final String interpreterRunner;
private final String interpreterDir;
private final String localRepoDir;
private GenericObjectPool<Client> clientPool;
private Map<String, String> env;
@ -53,20 +54,28 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int connectTimeout;
public RemoteInterpreterProcess(String intpRunner,
String intpDir,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener) {
this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(listener), connectTimeout);
String intpDir,
String localRepoDir,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener) {
this(intpRunner,
intpDir,
localRepoDir,
env,
new RemoteInterpreterEventPoller(listener),
connectTimeout);
}
RemoteInterpreterProcess(String intpRunner,
String intpDir,
String localRepoDir,
Map<String, String> env,
RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
this.interpreterRunner = intpRunner;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
this.env = env;
this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
referenceCount = new AtomicInteger(0);
@ -89,12 +98,13 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
throw new InterpreterException(e1);
}
CommandLine cmdLine = CommandLine.parse(interpreterRunner);
cmdLine.addArgument("-d", false);
cmdLine.addArgument(interpreterDir, false);
cmdLine.addArgument("-p", false);
cmdLine.addArgument(Integer.toString(port), false);
cmdLine.addArgument("-l", false);
cmdLine.addArgument(localRepoDir, false);
executor = new DefaultExecutor();
@ -139,6 +149,9 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
}
public Client getClient() throws Exception {
if (clientPool == null || clientPool.isClosed()) {
return null;
}
return clientPool.borrowObject();
}
@ -181,7 +194,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
} catch (Exception e) {
// safely ignore exception while client.shutdown() may terminates remote process
logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
"safely ignore exception while client.shutdown() may terminates remote process", e);
"safely ignore exception while client.shutdown() may terminates remote process");
logger.debug(e.getMessage(), e);
} finally {
if (client != null) {
// no longer used
@ -293,8 +307,13 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
} catch (TException e) {
broken = true;
logger.error("Can't update angular object", e);
} catch (NullPointerException e) {
logger.error("Remote interpreter process not started", e);
return;
} finally {
releaseClient(client, broken);
if (client != null) {
releaseClient(client, broken);
}
}
}

Some files were not shown because too many files have changed in this diff Show more