Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-1787

Browse source
This commit is contained in:
Alexander Shoshin 2017-01-09 14:58:14 +03:00
commit 5255e176ae
237 changed files with 8348 additions and 5795 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.gitignore vendored
View file

@ -39,11 +39,14 @@ reports
zeppelin-web/node_modules
zeppelin-web/dist
zeppelin-web/.tmp
zeppelin-web/src/fonts/Roboto*
zeppelin-web/src/fonts/Source-Code-Pro*
zeppelin-web/src/fonts/Patua-One*
zeppelin-web/src/fonts/roboto*
zeppelin-web/src/fonts/source-code-pro*
zeppelin-web/src/fonts/patua-one*
zeppelin-web/src/fonts/google-fonts.css
zeppelin-web/.sass-cache
zeppelin-web/npm-debug.log
zeppelin-web/bower_components
zeppelin-web/yarn.lock
**nbproject/
**node/

12
.travis.yml
View file

@ -33,8 +33,6 @@ addons:
- r-packages-precise
packages:
- r-base-dev
- r-cran-evaluate
- r-cran-base64enc
matrix:
include:
@ -46,6 +44,10 @@ matrix:
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with spark 2.1.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pbeam -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
@ -80,7 +82,7 @@ matrix:
before_install:
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
- ./testing/install_external_dependencies.sh > /dev/null 2>&1
- ./testing/install_external_dependencies.sh
- ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
@ -92,10 +94,8 @@ install:
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- if [[ -n $LIVY_VER ]]; then travis_retry ./testing/downloadLivy.sh $LIVY_VER; fi
- ./testing/setupLivy.sh
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
- if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
- tail conf/zeppelin-env.sh
script:

7
SECURITY-README.md
View file

@ -16,12 +16,7 @@ limitations under the License.
To connect to Zeppelin, users will be asked to enter their credentials. Once logged, a user has access to all notes including other users notes.
This a a first step toward full security as implemented by this pull request (https://github.com/apache/zeppelin/pull/53).
# Security setup
1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authc" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections.
2. Secure the Websocket channel : Set to property "zeppelin.anonymous.allowed" to "false" in the file conf/zeppelin-site.xml. You can start by renaming conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
3. Start Zeppelin : bin/zeppelin.sh
4. point your browser to http://localhost:8080
5. Login using one of the user/password combinations defined in the conf/shiro.ini file.
Please check [Shiro authentication in Apache Zeppelin](https://zeppelin.apache.org/docs/snapshot/security/shiroauthentication.html) in our official website for more detailed information(e.g. How to setup the security, How to configure user groups and permissions, and etc).
# Implementation notes
## Vocabulary

8
STYLE.md
View file

@ -25,11 +25,11 @@ bower.json
In the override section at the bottom, include the Highlightjs stylesheet (eg. styles/github.css)
For the selected Ace Editor theme script, include it in the override section. (eg. src-noconflict/theme-github.js)
(bower will automatically add the appropriate .js and .css in app/index.html)
```
```diff
"src-noconflict/mode-sql.js",
"src-noconflict/mode-markdown.js",
"src-noconflict/keybinding-emacs.js",
"src-noconflict/ext-language_tools.js",
"src-noconflict/ext-language_tools.js",
+ "src-noconflict/theme-github.js"],
"version": "1.1.8",
"name": "ace-builds"
@ -48,13 +48,13 @@ Highlight.js style - depends on the style, a few themes have jpg - if so, one mu
### Example - change Ace Editor theme to monokai
app/scripts/controllers/paragraph.js
```
```diff
- $scope.editor.setTheme('ace/theme/github');
+ $scope.editor.setTheme('ace/theme/monokai');
```
bower.json
```
```diff
- "src-noconflict/theme-github.js"],
+ "src-noconflict/theme-monokai.js"],
```

8
bin/common.cmd
View file

@ -43,14 +43,6 @@ if not defined ZEPPELIN_WAR (
)
)
if not defined ZEPPELIN_NOTEBOOK_DIR (
set ZEPPELIN_NOTEBOOK_DIR=%ZEPPELIN_HOME%\notebook
)
if not defined ZEPPELIN_INTERPRETER_DIR (
set ZEPPELIN_INTERPRETER_DIR=%ZEPPELIN_HOME%\interpreter
)
if exist "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" (
call "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd"
)

8
bin/common.sh
View file

@ -48,14 +48,6 @@ if [[ -z "${ZEPPELIN_WAR}" ]]; then
fi
fi
if [[ -z "$ZEPPELIN_NOTEBOOK_DIR" ]]; then
export ZEPPELIN_NOTEBOOK_DIR="${ZEPPELIN_HOME}/notebook"
fi
if [[ -z "$ZEPPELIN_INTERPRETER_DIR" ]]; then
export ZEPPELIN_INTERPRETER_DIR="${ZEPPELIN_HOME}/interpreter"
fi
if [[ -f "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh" ]]; then
. "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh"
fi

5
bin/zeppelin.cmd
View file

@ -84,9 +84,4 @@ if not exist %ZEPPELIN_PID_DIR% (
mkdir "%ZEPPELIN_PID_DIR%"
)
if not exist %ZEPPELIN_NOTEBOOK_DIR% (
echo Notebook dir doesn't exist, create %ZEPPELIN_NOTEBOOK_DIR%
mkdir "%ZEPPELIN_NOTEBOOK_DIR%"
)
"%ZEPPELIN_RUNNER%" %JAVA_OPTS% -cp %CLASSPATH% %ZEPPELIN_SERVER% "%*"

5
bin/zeppelin.sh
View file

@ -83,9 +83,4 @@ if [[ ! -d "${ZEPPELIN_PID_DIR}" ]]; then
$(mkdir -p "${ZEPPELIN_PID_DIR}")
fi
if [[ ! -d "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
echo "Pid dir doesn't exist, create ${ZEPPELIN_NOTEBOOK_DIR}"
$(mkdir -p "${ZEPPELIN_NOTEBOOK_DIR}")
fi
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_SERVER "$@"

6
cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java
View file

@ -226,10 +226,4 @@ public class CassandraInterpreter extends Interpreter {
.createOrGetParallelScheduler(CassandraInterpreter.class.getName() + this.hashCode(),
parseInt(getProperty(CASSANDRA_INTERPRETER_PARALLELISM)));
}
@Override
public void destroy() {
super.destroy();
this.close();
}
}

15
conf/README.md
View file

@ -1,15 +0,0 @@
## Enabling SSL
Enabling SSL requires a few changes. The first is to set zeppelin.ssl to true. If you'll like to use client side certificate authentication as well, then set zeppelin.ssl.client.auth to true too.
Information how about to generate certificates and a keystore can be found [here](https://wiki.eclipse.org/Jetty/Howto/Configure_SSL).
A condensed example can be found in the top answer to this [StackOverflow post](http://stackoverflow.com/questions/4008837/configure-ssl-on-jetty).
The keystore holds the private key and certificate on the server end. The trustore holds the trusted client certificates. Be sure that the path and password for these two stores are correctly configured in the password fields below. They can be obfuscated using the Jetty password tool. After Maven pulls in all the dependency to build Zeppelin, one of the Jetty jars contain the Password tool. Invoke this command from the Zeppelin home build directory with the appropriate version, user, and password.
```
java -cp ./zeppelin-server/target/lib/jetty-all-server-<version>.jar org.eclipse.jetty.util.security.Password <user> <password>
```
If you are using a self-signed, a certificate signed by an untrusted CA, or if client authentication is enabled, then the client must have a browser create exceptions for both the normal HTTPS port and WebSocket port. This can by done by trying to establish an HTTPS connection to both ports in a browser (i.e. if the ports are 443 and 8443, then visit https://127.0.0.1:443 and https://127.0.0.1:8443). This step can be skipped if the server certificate is signed by a trusted CA and client auth is disabled.

16
conf/zeppelin-site.xml.template
View file

@ -147,11 +147,11 @@
</property>
-->
<!-- For versioning your local notebook storage using Git repository
<!-- Notebook storage layer using local file system
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
<description>notebook persistence layer implementation</description>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
<description>local notebook persistence layer implementation</description>
</property>
-->
@ -159,15 +159,15 @@
<!--
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo</value>
<description>two notebook persistence layers (local + ZeppelinHub)</description>
<value>org.apache.zeppelin.notebook.repo.GitNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo</value>
<description>two notebook persistence layers (versioned local + ZeppelinHub)</description>
</property>
-->
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
<description>notebook persistence layer implementation</description>
<value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
<description>versioned notebook persistence layer implementation</description>
</property>
<property>
@ -190,7 +190,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

28
dev/create_release.sh
View file

@ -33,14 +33,28 @@ if [[ $# -ne 2 ]]; then
usage
fi
if [[ -z "${GPG_PASSPHRASE}" ]]; then
echo "You need GPG_PASSPHRASE variable set"
exit 1
fi
for var in GPG_PASSPHRASE DOCKER_USERNAME; do
if [[ -z "${!var}" ]]; then
echo "You need ${var} variable set"
exit 1
fi
done
RELEASE_VERSION="$1"
GIT_TAG="$2"
function build_docker_base() {
# build base image
docker build -t ${DOCKER_USERNAME}/zeppelin-base:latest "${BASEDIR}/../scripts/docker/zeppelin-base"
}
function build_docker_image() {
# build release image
echo "FROM ${DOCKER_USERNAME}/zeppelin-base:latest
RUN mkdir /usr/local/zeppelin/
ADD zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} /usr/local/zeppelin/" > "Dockerfile"
docker build -t ${DOCKER_USERNAME}/zeppelin-release:"${RELEASE_VERSION}" .
}
function make_source_package() {
# create source package
cd ${WORKING_DIR}
@ -97,10 +111,16 @@ function make_binary_release() {
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5" "${WORKING_DIR}/"
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512" "${WORKING_DIR}/"
# build docker image if binary_release_name 'all'
if [[ $1 = "all" ]]; then
build_docker_image
fi
# clean up build dir
rm -rf "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
}
build_docker_base
git_clone
make_source_package
make_binary_release all "-Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pscala-2.11"

11
dev/publish_release.sh
View file

@ -30,7 +30,7 @@ if [[ $# -ne 2 ]]; then
usage
fi
for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do
for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD DOCKER_USERNAME DOCKER_PASSWORD DOCKER_EMAIL; do
if [[ -z "${!var}" ]]; then
echo "You need ${var} variable set"
exit 1
@ -67,6 +67,14 @@ function curl_error() {
fi
}
function publish_to_dockerhub() {
# publish images
docker login --username="${DOCKER_USERNAME}" --password="${DOCKER_PASSWORD}" --email="${DOCKER_EMAIL}"
docker push ${DOCKER_USERNAME}/zeppelin-base:latest
docker push ${DOCKER_USERNAME}/zeppelin-release:"${RELEASE_VERSION}"
}
function publish_to_maven() {
cd "${WORKING_DIR}/zeppelin"
@ -153,5 +161,6 @@ function publish_to_maven() {
}
git_clone
publish_to_dockerhub
publish_to_maven
cleanup

3
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -22,7 +22,7 @@
<li role="separator" class="divider"></li>
<li class="title"><span><b>Getting Started</b><span></li>
<li><a href="{{BASE_PATH}}/install/install.html">Install</a></li>
<li><a href="{{BASE_PATH}}/install/install.html#apache-zeppelin-configuration">Configuration</a></li>
<li><a href="{{BASE_PATH}}/install/configuration.html">Configuration</a></li>
<li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li>
<li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li>
<li role="separator" class="divider"></li>
@ -47,6 +47,7 @@
<!--<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>-->
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
<li><a href="{{BASE_PATH}}/manual/userimpersonation.html">Interpreter User Impersonation</a></li>
<li><a href="{{BASE_PATH}}/manual/interpreterexechooks.html">Interpreter Execution Hooks (Experimental)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Available Interpreters</b><span></li>
<li><a href="{{BASE_PATH}}/interpreter/alluxio.html">Alluxio</a></li>

16
docs/displaysystem/basicdisplaysystem.md
View file

@ -27,37 +27,37 @@ limitations under the License.
By default, Apache Zeppelin prints interpreter response as a plain text using `text` display system.
<img src="/assets/themes/zeppelin/img/screenshots/display_text.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_text.png" />
You can explicitly say you're using `text` display system.
<img src="/assets/themes/zeppelin/img/screenshots/display_text1.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_text1.png" />
## Html
With `%html` directive, Zeppelin treats your output as HTML
<img src="/assets/themes/zeppelin/img/screenshots/display_html.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_html.png" />
### Mathematical expressions
HTML display system automatically formats mathematical expression using [MathJax](https://www.mathjax.org/). You can use
`\\( INLINE EXPRESSION \\)` and `$$ EXPRESSION $$` to format. For example
<img src="/assets/themes/zeppelin/img/screenshots/display_formula.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_formula.png" />
## Table
If you have data that row separated by '\n' (newline) and column separated by '\t' (tab) with first row as header row, for example
If you have data that row separated by `\n` (newline) and column separated by `\t` (tab) with first row as header row, for example
<img src="/assets/themes/zeppelin/img/screenshots/display_table.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_table.png" />
You can simply use `%table` display system to leverage Zeppelin's built in visualization.
<img src="/assets/themes/zeppelin/img/screenshots/display_table1.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_table1.png" />
If table contents start with `%html`, it is interpreted as an HTML.
<img src="/assets/themes/zeppelin/img/screenshots/display_table_html.png" />
<img src="../assets/themes/zeppelin/img/screenshots/display_table_html.png" />
> **Note :** Display system is backend independent.

3
docs/index.md
View file

@ -125,7 +125,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* Getting Started
* [Quick Start](./install/install.html) for basic instructions on installing Apache Zeppelin
* [Configuration](./install/install.html#apache-zeppelin-configuration) lists for Apache Zeppelin
* [Configuration](./install/configuration.html) lists for Apache Zeppelin
* [Explore Apache Zeppelin UI](./quickstart/explorezeppelinui.html): basic components of Apache Zeppelin home
* [Tutorial](./quickstart/tutorial.html): a short walk-through tutorial that uses Apache Spark backend
* Basic Feature Guide
@ -143,6 +143,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* [Interpreter Installation](./manual/interpreterinstallation.html): Install not only community managed interpreters but also 3rd party interpreters
* [Interpreter Dependency Management](./manual/dependencymanagement.html) when you include external libraries to interpreter
* [Interpreter User Impersonation](./manual/userimpersonation.html) when you want to run interpreter as end user
* [Interpreter Execution Hooks](./manual/interpreterexechooks.html) to specify additional code to be executed by an interpreter at pre and post-paragraph code execution
* Available Interpreters: currently, about 20 interpreters are available in Apache Zeppelin.
####Display System

2
docs/install/build.md
View file

@ -205,7 +205,7 @@ mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
Ignite Interpreter
```bash
mvn clean package -Dignite.version=1.6.0 -DskipTests
mvn clean package -Dignite.version=1.8.0 -DskipTests
```
Scalding Interpreter

388
docs/install/configuration.md Normal file
View file

@ -0,0 +1,388 @@
---
layout: page
title: "Apache Zeppelin Configuration"
description: "This page will guide you to configure Apache Zeppelin using either environment variables or Java properties. Also, you can configure SSL for Zeppelin."
group: install
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Apache Zeppelin Configuration
<div id="toc"></div>
## Zeppelin Properties
There are two locations you can configure Apache Zeppelin.
* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
* **Java properties** can ba defined in `conf/zeppelin-site.xml`.
If both are defined, then the **environment variables** will take priority.
<table class="table-configuration">
<tr>
<th>zeppelin-env.sh</th>
<th>zeppelin-site.xml</th>
<th>Default value</th>
<th class="col-md-4">Description</th>
</tr>
<tr>
<td>ZEPPELIN_PORT</td>
<td>zeppelin.server.port</td>
<td>8080</td>
<td>Zeppelin server port</td>
</tr>
<tr>
<td>ZEPPELIN_SSL_PORT</td>
<td>zeppelin.server.ssl.port</td>
<td>8443</td>
<td>Zeppelin Server ssl port (used when ssl environment/property is set to true)</td>
</tr>
<tr>
<td>ZEPPELIN_MEM</td>
<td>N/A</td>
<td>-Xmx1024m -XX:MaxPermSize=512m</td>
<td>JVM mem options</td>
</tr>
<tr>
<td>ZEPPELIN_INTP_MEM</td>
<td>N/A</td>
<td>ZEPPELIN_MEM</td>
<td>JVM mem options for interpreter process</td>
</tr>
<tr>
<td>ZEPPELIN_JAVA_OPTS</td>
<td>N/A</td>
<td></td>
<td>JVM options</td>
</tr>
<tr>
<td>ZEPPELIN_ALLOWED_ORIGINS</td>
<td>zeppelin.server.allowed.origins</td>
<td>*</td>
<td>Enables a way to specify a ',' separated list of allowed origins for REST and websockets. <br /> e.g. http://localhost:8080 </td>
</tr>
<tr>
<td>N/A</td>
<td>zeppelin.anonymous.allowed</td>
<td>true</td>
<td>The anonymous user is allowed by default.</td>
</tr>
<tr>
<td>ZEPPELIN_SERVER_CONTEXT_PATH</td>
<td>zeppelin.server.context.path</td>
<td>/</td>
<td>Context path of the web application</td>
</tr>
<tr>
<td>ZEPPELIN_SSL</td>
<td>zeppelin.ssl</td>
<td>false</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_CLIENT_AUTH</td>
<td>zeppelin.ssl.client.auth</td>
<td>false</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_PATH</td>
<td>zeppelin.ssl.keystore.path</td>
<td>keystore</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_TYPE</td>
<td>zeppelin.ssl.keystore.type</td>
<td>JKS</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_PASSWORD</td>
<td>zeppelin.ssl.keystore.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEY_MANAGER_PASSWORD</td>
<td>zeppelin.ssl.key.manager.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_PATH</td>
<td>zeppelin.ssl.truststore.path</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_TYPE</td>
<td>zeppelin.ssl.truststore.type</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_PASSWORD</td>
<td>zeppelin.ssl.truststore.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN</td>
<td>zeppelin.notebook.homescreen</td>
<td></td>
<td>Display note IDs on the Apache Zeppelin homescreen <br />e.g. 2A94M5J1Z</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE</td>
<td>zeppelin.notebook.homescreen.hide</td>
<td>false</td>
<td>Hide the note ID set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Apache Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
</tr>
<tr>
<td>ZEPPELIN_WAR_TEMPDIR</td>
<td>zeppelin.war.tempdir</td>
<td>webapps</td>
<td>Location of the jetty temporary directory</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_DIR</td>
<td>zeppelin.notebook.dir</td>
<td>notebook</td>
<td>The root directory where notebook directories are saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_BUCKET</td>
<td>zeppelin.notebook.s3.bucket</td>
<td>zeppelin</td>
<td>S3 Bucket where notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_USER</td>
<td>zeppelin.notebook.s3.user</td>
<td>user</td>
<td>User name of an S3 bucket<br />e.g. <code>bucket/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_ENDPOINT</td>
<td>zeppelin.notebook.s3.endpoint</td>
<td>s3.amazonaws.com</td>
<td>Endpoint for the bucket</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID</td>
<td>zeppelin.notebook.s3.kmsKeyID</td>
<td></td>
<td>AWS KMS Key ID to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_EMP</td>
<td>zeppelin.notebook.s3.encryptionMaterialsProvider</td>
<td></td>
<td>Class name of a custom S3 encryption materials provider implementation to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />e.g. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=&lt;accountName&gt;;<br/>AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Azure Share where the notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
<td>zeppelin.notebook.azure.user</td>
<td>user</td>
<td>Optional user name of an Azure file share<br />e.g. <code>share/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_STORAGE</td>
<td>zeppelin.notebook.storage</td>
<td>org.apache.zeppelin.notebook.repo.GitNotebookRepo</td>
<td>Comma separated list of notebook storage locations</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC</td>
<td>zeppelin.notebook.one.way.sync</td>
<td>false</td>
<td>If there are multiple notebook storage locations, should we treat the first one as the only source of truth?</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_PUBLIC</td>
<td>zeppelin.notebook.public</td>
<td>true</td>
<td>Make notebook public (set only <code>owners</code>) by default when created/imported. If set to <code>false</code> will add <code>user</code> to <code>readers</code> and <code>writers</code> as well, making it private and invisible to other users unless permissions are granted.</td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETERS</td>
<td>zeppelin.interpreters</td>
<description></description>
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />
...
</td>
<td>
Comma separated interpreter configurations [Class] <br/><br />
<span style="font-style:italic; color: gray">NOTE: This property is deprecated since Zeppelin-0.6.0 and will not be supported from Zeppelin-0.7.0.</span>
</td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETER_DIR</td>
<td>zeppelin.interpreter.dir</td>
<td>interpreter</td>
<td>Interpreter directory</td>
</tr>
<tr>
<td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td>
<td>zeppelin.websocket.max.text.message.size</td>
<td>1024000</td>
<td>Size (in characters) of the maximum text message that can be received by websocket.</td>
</tr>
</table>
## SSL Configuration
Enabling SSL requires a few configuration changes. First, you need to create certificates and then update necessary configurations to enable server side SSL and/or client side certificate authentication.
### Creating and configuring the Certificates
Information how about to generate certificates and a keystore can be found [here](https://wiki.eclipse.org/Jetty/Howto/Configure_SSL).
A condensed example can be found in the top answer to this [StackOverflow post](http://stackoverflow.com/questions/4008837/configure-ssl-on-jetty).
The keystore holds the private key and certificate on the server end. The trustore holds the trusted client certificates. Be sure that the path and password for these two stores are correctly configured in the password fields below. They can be obfuscated using the Jetty password tool. After Maven pulls in all the dependency to build Zeppelin, one of the Jetty jars contain the Password tool. Invoke this command from the Zeppelin home build directory with the appropriate version, user, and password.
```
java -cp ./zeppelin-server/target/lib/jetty-all-server-<version>.jar org.eclipse.jetty.util.security.Password <user> <password>
```
If you are using a self-signed, a certificate signed by an untrusted CA, or if client authentication is enabled, then the client must have a browser create exceptions for both the normal HTTPS port and WebSocket port. This can by done by trying to establish an HTTPS connection to both ports in a browser (e.g. if the ports are 443 and 8443, then visit https://127.0.0.1:443 and https://127.0.0.1:8443). This step can be skipped if the server certificate is signed by a trusted CA and client auth is disabled.
### Configuring server side SSL
The following properties needs to be updated in the `zeppelin-site.xml` in order to enable server side SSL.
```
<property>
<name>zeppelin.server.ssl.port</name>
<value>8443</value>
<description>Server ssl port. (used when ssl property is set to true)</description>
</property>
<property>
<name>zeppelin.ssl</name>
<value>true</value>
<description>Should SSL be used by the servers?</description>
</property>
<property>
<name>zeppelin.ssl.keystore.path</name>
<value>keystore</value>
<description>Path to keystore relative to Zeppelin configuration directory</description>
</property>
<property>
<name>zeppelin.ssl.keystore.type</name>
<value>JKS</value>
<description>The format of the given keystore (e.g. JKS or PKCS12)</description>
</property>
<property>
<name>zeppelin.ssl.keystore.password</name>
<value>change me</value>
<description>Keystore password. Can be obfuscated by the Jetty Password tool</description>
</property>
<property>
<name>zeppelin.ssl.key.manager.password</name>
<value>change me</value>
<description>Key Manager password. Defaults to keystore password. Can be obfuscated.</description>
</property>
```
### Enabling client side certificate authentication
The following properties needs to be updated in the `zeppelin-site.xml` in order to enable client side certificate authentication.
```
<property>
<name>zeppelin.server.ssl.port</name>
<value>8443</value>
<description>Server ssl port. (used when ssl property is set to true)</description>
</property>
<property>
<name>zeppelin.ssl.client.auth</name>
<value>true</value>
<description>Should client authentication be used for SSL connections?</description>
</property>
<property>
<name>zeppelin.ssl.truststore.path</name>
<value>truststore</value>
<description>Path to truststore relative to Zeppelin configuration directory. Defaults to the keystore path</description>
</property>
<property>
<name>zeppelin.ssl.truststore.type</name>
<value>JKS</value>
<description>The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type</description>
</property>
<property>
<name>zeppelin.ssl.truststore.password</name>
<value>change me</value>
<description>Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password</description>
</property>
```
### Obfuscating Passwords using the Jetty Password Tool
Security best practices advise to not use plain text passwords and Jetty provides a password tool to help obfuscating the passwords used to access the KeyStore and TrustStore.
The Password tool documentation can be found [here](http://www.eclipse.org/jetty/documentation/current/configuring-security-secure-passwords.html).
After using the tool:
```
java -cp $ZEPPELIN_HOME/zeppelin-server/target/lib/jetty-util-9.2.15.v20160210.jar \
org.eclipse.jetty.util.security.Password \
password
2016-12-15 10:46:47.931:INFO::main: Logging initialized @101ms
password
OBF:1v2j1uum1xtv1zej1zer1xtn1uvk1v1v
MD5:5f4dcc3b5aa765d61d8327deb882cf99
```
update your configuration with the obfuscated password :
```
<property>
<name>zeppelin.ssl.keystore.password</name>
<value>OBF:1v2j1uum1xtv1zej1zer1xtn1uvk1v1v</value>
<description>Keystore password. Can be obfuscated by the Jetty Password tool</description>
</property>
```
**Note:** After updating these configurations, Zeppelin server needs to be restarted.

70
docs/install/docker.md Normal file
View file

@ -0,0 +1,70 @@
---
layout: page
title: "Apache Zeppelin Releases Docker Images"
description: "This document contains instructions about making docker containers for Zeppelin. It mainly provides guidance into how to create, publish and run docker images for zeppelin releases."
group: install
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Apache Zeppelin Releases Docker Images
<div id="toc"></div>
## Overview
This document contains instructions about making docker containers for Zeppelin. It mainly provides guidance into how to create, publish and run docker images for zeppelin releases.
## Quick Start
### Installing Docker
You need to [install docker](https://docs.docker.com/engine/installation/) on your machine.
### Creating and Publishing Zeppelin docker image
* In order to be able to create and/or publish an image, you need to set the **DockerHub** credentials `DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_EMAIL` variables as environment variables.
* To create an image for some release use :
`create_release.sh <release-version> <git-tag>`.
* To publish the created image use :
`publish_release.sh <release-version> <git-tag>`
### Running a Zeppelin docker image
* To start Zeppelin, you need to pull the zeppelin release image:
```
docker pull ${DOCKER_USERNAME}/zeppelin-release:<release-version>
docker run --rm -it -p 7077:7077 -p 8080:8080 ${DOCKER_USERNAME}/zeppelin-release:<release-version> -c bash
```
* Then a docker container will start with a Zeppelin release on path :
`/usr/local/zeppelin/`
* Run zeppelin inside docker:
```
/usr/local/zeppelin/bin/zeppelin.sh
```
* To Run Zeppelin in daemon mode
Mounting logs and notebooks zeppelin to folders on your host machine
```
docker run -p 7077:7077 -p 8080:8080 --privileged=true -v $PWD/logs:/logs -v $PWD/notebook:/notebook \
-e ZEPPELIN_NOTEBOOK_DIR='/notebook' \
-e ZEPPELIN_LOG_DIR='/logs' \
-d ${DOCKER_USERNAME}/zeppelin-release:<release-version> \
/usr/local/zeppelin/bin/zeppelin.sh
```
* Zeppelin will run at `http://localhost:8080`.

300
docs/install/install.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Quick Start"
description: "This page will help you get started and will guide you through installing Apache Zeppelin, running it in the command line and configuring options."
description: "This page will help you get started and will guide you through installing Apache Zeppelin and running it in the command line."
group: install
---
<!--
@ -56,8 +56,9 @@ Two binary packages are available on the [Apache Zeppelin Download Page](http://
Unpack and follow [install additional interpreters](../manual/interpreterinstallation.html) to install interpreters. If you're unsure, just run `./bin/install-interpreter.sh --all` and install all interpreters.
## Starting Apache Zeppelin from the Command Line
#### Starting Apache Zeppelin
## Starting Apache Zeppelin
#### Starting Apache Zeppelin from the Command Line
On all unix like platforms:
@ -79,266 +80,6 @@ After Zeppelin has started successfully, go to [http://localhost:8080](http://lo
bin/zeppelin-daemon.sh stop
```
## Next Steps
Congratulations, you have successfully installed Apache Zeppelin! Here are few steps you might find useful:
#### New to Apache Zeppelin...
* For an in-depth overview, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html).
* And then, try run [tutorial](http://localhost:8080/#/notebook/2A94M5J1Z) notebook in your Zeppelin.
* And see how to change [configurations](#apache-zeppelin-configuration) like port number, etc.
#### Zeppelin with Apache Spark ...
* To know more about deep integration with [Apache Spark](http://spark.apache.org/), check [Spark Interpreter](../interpreter/spark.html).
#### Zeppelin with JDBC data sources ...
* Check [JDBC Interpreter](../interpreter/jdbc.html) to know more about configure and uses multiple JDBC data sources.
#### Zeppelin with Python ...
* Check [Python interpreter](../interpreter/python.html) to know more about Matplotlib, Pandas, Conda/Docker environment integration.
#### Multi-user environment ...
* Turn on [authentication](../security/shiroauthentication.html).
* Manage your [notebook permission](../security/notebook_authorization.html).
* For more informations, go to **More** -> **Security** section.
#### Other useful informations ...
* Learn how [Display System](../displaysystem/basicdisplaysystem.html) works.
* Use [Service Manager](#start-apache-zeppelin-with-a-service-manager) to start Zeppelin.
* If you're using previous version please see [Upgrade Zeppelin version](./upgrade.html).
## Apache Zeppelin Configuration
You can configure Apache Zeppelin with either **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) or **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
<table class="table-configuration">
<tr>
<th>zeppelin-env.sh</th>
<th>zeppelin-site.xml</th>
<th>Default value</th>
<th class="col-md-4">Description</th>
</tr>
<tr>
<td>ZEPPELIN_PORT</td>
<td>zeppelin.server.port</td>
<td>8080</td>
<td>Zeppelin server port</td>
</tr>
<tr>
<td>ZEPPELIN_SSL_PORT</td>
<td>zeppelin.server.ssl.port</td>
<td>8443</td>
<td>Zeppelin Server ssl port (used when ssl environment/property is set to true)</td>
</tr>
<tr>
<td>ZEPPELIN_MEM</td>
<td>N/A</td>
<td>-Xmx1024m -XX:MaxPermSize=512m</td>
<td>JVM mem options</td>
</tr>
<tr>
<td>ZEPPELIN_INTP_MEM</td>
<td>N/A</td>
<td>ZEPPELIN_MEM</td>
<td>JVM mem options for interpreter process</td>
</tr>
<tr>
<td>ZEPPELIN_JAVA_OPTS</td>
<td>N/A</td>
<td></td>
<td>JVM options</td>
</tr>
<tr>
<td>ZEPPELIN_ALLOWED_ORIGINS</td>
<td>zeppelin.server.allowed.origins</td>
<td>*</td>
<td>Enables a way to specify a ',' separated list of allowed origins for REST and websockets. <br /> i.e. http://localhost:8080 </td>
</tr>
<tr>
<td>N/A</td>
<td>zeppelin.anonymous.allowed</td>
<td>true</td>
<td>The anonymous user is allowed by default.</td>
</tr>
<tr>
<td>ZEPPELIN_SERVER_CONTEXT_PATH</td>
<td>zeppelin.server.context.path</td>
<td>/</td>
<td>Context path of the web application</td>
</tr>
<tr>
<td>ZEPPELIN_SSL</td>
<td>zeppelin.ssl</td>
<td>false</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_CLIENT_AUTH</td>
<td>zeppelin.ssl.client.auth</td>
<td>false</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_PATH</td>
<td>zeppelin.ssl.keystore.path</td>
<td>keystore</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_TYPE</td>
<td>zeppelin.ssl.keystore.type</td>
<td>JKS</td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEYSTORE_PASSWORD</td>
<td>zeppelin.ssl.keystore.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_KEY_MANAGER_PASSWORD</td>
<td>zeppelin.ssl.key.manager.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_PATH</td>
<td>zeppelin.ssl.truststore.path</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_TYPE</td>
<td>zeppelin.ssl.truststore.type</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_SSL_TRUSTSTORE_PASSWORD</td>
<td>zeppelin.ssl.truststore.password</td>
<td></td>
<td></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN</td>
<td>zeppelin.notebook.homescreen</td>
<td></td>
<td>Display note IDs on the Apache Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE</td>
<td>zeppelin.notebook.homescreen.hide</td>
<td>false</td>
<td>Hide the note ID set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Apache Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
</tr>
<tr>
<td>ZEPPELIN_WAR_TEMPDIR</td>
<td>zeppelin.war.tempdir</td>
<td>webapps</td>
<td>Location of the jetty temporary directory</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_DIR</td>
<td>zeppelin.notebook.dir</td>
<td>notebook</td>
<td>The root directory where notebook directories are saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_BUCKET</td>
<td>zeppelin.notebook.s3.bucket</td>
<td>zeppelin</td>
<td>S3 Bucket where notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_USER</td>
<td>zeppelin.notebook.s3.user</td>
<td>user</td>
<td>User name of an S3 bucket<br />i.e. <code>bucket/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_ENDPOINT</td>
<td>zeppelin.notebook.s3.endpoint</td>
<td>s3.amazonaws.com</td>
<td>Endpoint for the bucket</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID</td>
<td>zeppelin.notebook.s3.kmsKeyID</td>
<td></td>
<td>AWS KMS Key ID to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_EMP</td>
<td>zeppelin.notebook.s3.encryptionMaterialsProvider</td>
<td></td>
<td>Class name of a custom S3 encryption materials provider implementation to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
<td></td>
<td>The Azure storage account connection string<br />i.e. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=&lt;accountName&gt;;<br/>AccountKey=&lt;accountKey&gt;</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Azure Share where the notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
<td>zeppelin.notebook.azure.user</td>
<td>user</td>
<td>Optional user name of an Azure file share<br />i.e. <code>share/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_STORAGE</td>
<td>zeppelin.notebook.storage</td>
<td>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</td>
<td>Comma separated list of notebook storage locations</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC</td>
<td>zeppelin.notebook.one.way.sync</td>
<td>false</td>
<td>If there are multiple notebook storage locations, should we treat the first one as the only source of truth?</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_PUBLIC</td>
<td>zeppelin.notebook.public</td>
<td>true</td>
<td>Make notebook public (set only `owners`) by default when created/imported. If set to `false` will add `user` to `readers` and `writers` as well, making it private and invisible to other users unless permissions are granted.</td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETERS</td>
<td>zeppelin.interpreters</td>
<description></description>
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />
...
</td>
<td>
Comma separated interpreter configurations [Class] <br/>
<span style="font-style:italic">NOTE: This property is deprecated since Zeppelin-0.6.0 and will not be supported from Zeppelin-0.7.0 on.</span>
</td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETER_DIR</td>
<td>zeppelin.interpreter.dir</td>
<td>interpreter</td>
<td>Interpreter directory</td>
</tr>
<tr>
<td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td>
<td>zeppelin.websocket.max.text.message.size</td>
<td>1024000</td>
<td>Size (in characters) of the maximum text message that can be received by websocket.</td>
</tr>
</table>
#### Start Apache Zeppelin with a service manager
> **Note :** The below description was written based on Ubuntu Linux.
@ -381,6 +122,37 @@ exec bin/zeppelin-daemon.sh upstart
```
## Building from Source
## Next Steps
Congratulations, you have successfully installed Apache Zeppelin! Here are few steps you might find useful:
#### New to Apache Zeppelin...
* For an in-depth overview, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html).
* And then, try run [tutorial](http://localhost:8080/#/notebook/2A94M5J1Z) notebook in your Zeppelin.
* And see how to change [configurations](./configuration.html) like port number, etc.
#### Zeppelin with Apache Spark ...
* To know more about deep integration with [Apache Spark](http://spark.apache.org/), check [Spark Interpreter](../interpreter/spark.html).
#### Zeppelin with JDBC data sources ...
* Check [JDBC Interpreter](../interpreter/jdbc.html) to know more about configure and uses multiple JDBC data sources.
#### Zeppelin with Python ...
* Check [Python interpreter](../interpreter/python.html) to know more about Matplotlib, Pandas, Conda/Docker environment integration.
#### Multi-user environment ...
* Turn on [authentication](../security/shiroauthentication.html).
* Manage your [notebook permission](../security/notebook_authorization.html).
* For more informations, go to **More** -> **Security** section.
#### Other useful informations ...
* Learn how [Display System](../displaysystem/basicdisplaysystem.html) works.
* Use [Service Manager](#start-apache-zeppelin-with-a-service-manager) to start Zeppelin.
* If you're using previous version please see [Upgrade Zeppelin version](./upgrade.html).
## Building Apache Zeppelin from Source
If you want to build from source instead of using binary package, follow the instructions [here](./build.html).

27
docs/install/spark_cluster_mode.md
View file

@ -208,3 +208,30 @@ Don't forget to set Spark `master` as `mesos://127.0.1.1:5050` in Zeppelin **Int
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:5050/#/frameworks` and check Zeppelin application is running well or not.
<img src="../assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" />
### Troubleshooting for Spark on Mesos
- If you have problem with hostname, use `--add-host` option when executing `dockerrun`
```
## use `--add-host=moby:127.0.0.1` option to resolve
## since docker container couldn't resolve `moby`
: java.net.UnknownHostException: moby: moby: Name or service not known
at java.net.InetAddress.getLocalHost(InetAddress.java:1496)
at org.apache.spark.util.Utils$.findLocalInetAddress(Utils.scala:789)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress$lzycompute(Utils.scala:782)
at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress(Utils.scala:782)
```
- If you have problem with mesos master, try `mesos://127.0.0.1` instead of `mesos://127.0.1.1`
```
I0103 20:17:22.329269 340 sched.cpp:330] New master detected at master@127.0.1.1:5050
I0103 20:17:22.330749 340 sched.cpp:341] No credentials provided. Attempting to register without authentication
W0103 20:17:22.333531 340 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050'
W0103 20:17:24.040252 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050'
W0103 20:17:26.150250 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050'
W0103 20:17:26.737604 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050'
W0103 20:17:35.241714 336 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050'
```

3
docs/install/upgrade.md
View file

@ -54,4 +54,5 @@ So, copying `notebook` and `conf` directory should be enough.
- Usage of `ZEPPELIN_PORT` is not supported in ssl mode. Instead use `ZEPPELIN_SSL_PORT` to configure the ssl port. Value from `ZEPPELIN_PORT` is used only when `ZEPPELIN_SSL` is set to `false`.
- The support on Spark 1.1.x to 1.3.x is deprecated.
- From 0.7, we uses `pegdown` as the `markdown.parser.type` option for the `%md` interpreter. Rendered markdown might be different from what you expected
- From 0.7 note.json format has been changed to support multiple outputs in a paragraph. Zeppelin will automatically convert old format to new format. 0.6 or lower version can read new note.json format but output will not be displayed. For the detail, see [ZEPPELIN-212](http://issues.apache.org/jira/browse/ZEPPELIN-212) and [pullrequest](https://github.com/apache/zeppelin/pull/1658).
- From 0.7 note.json format has been changed to support multiple outputs in a paragraph. Zeppelin will automatically convert old format to new format. 0.6 or lower version can read new note.json format but output will not be displayed. For the detail, see [ZEPPELIN-212](http://issues.apache.org/jira/browse/ZEPPELIN-212) and [pull request](https://github.com/apache/zeppelin/pull/1658).
- From 0.7 note storage layer will utilize `GitNotebookRepo` by default instead of `VFSNotebookRepo` storage layer, which is an extension of latter one with versioning capabilities on top of it.

2
docs/interpreter/flink.md
View file

@ -63,7 +63,7 @@ wget http://www.gutenberg.org/ebooks/10.txt.utf-8
{% highlight scala %}
%flink
case class WordCount(word: String, frequency: Int)
val bible:DataSet[String] = env.readTextFile("10.txt.utf-8")
val bible:DataSet[String] = benv.readTextFile("10.txt.utf-8")
val partialCounts: DataSet[WordCount] = bible.flatMap{
line =>
"""\b\w+\b""".r.findAllIn(line).map(word => WordCount(word, 1))

10
docs/interpreter/kylin.md
View file

@ -29,7 +29,6 @@ limitations under the License.
To get start with Apache Kylin, please see [Apache Kylin Quickstart](https://kylin.apache.org/docs15/index.html).
## Configuration
<table class="table-configuration">
<table class="table-configuration">
<tr>
<th>Name</th>
@ -54,7 +53,7 @@ To get start with Apache Kylin, please see [Apache Kylin Quickstart](https://kyl
<tr>
<td>kylin.query.project</td>
<td>learn_kylin</td>
<td>String, Project to perform query.</td>
<td>String, Project to perform query. Could update at notebook level</td>
</tr>
<tr>
<td>kylin.query.ispartial</td>
@ -72,15 +71,12 @@ To get start with Apache Kylin, please see [Apache Kylin Quickstart](https://kyl
<td>int, Query offset <br/> If offset is set in sql, curIndex will be ignored.</td>
</tr>
</table>
</table>
## Using the Apache Kylin Interpreter
In a paragraph, use `%kylin` to select the **kylin** interpreter and then input sql.
In a paragraph, use `%kylin(project_name)` to select the **kylin** interpreter, **project name** and then input **sql**. If no project name defined, will use the default project name from the above configuration.
```
%kylin
%kylin(learn_project)
select count(*) from kylin_sales group by part_dt
```

11
docs/interpreter/livy.md
View file

@ -40,9 +40,9 @@ Additional requirements for the Livy interpreter are:
## Configuration
We added some common configurations for spark, and you can set any configuration you want.
This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties.
You can find all Spark configurations in [here](http://spark.apache.org/docs/latest/configuration.html#available-properties).
And instead of starting property with `spark.` it should be replaced with `livy.spark.`.
Example: `spark.master` to `livy.spark.master`
Example: `spark.driver.memory` to `livy.spark.driver.memory`
<table class="table-configuration">
<tr>
@ -50,11 +50,6 @@ Example: `spark.master` to `livy.spark.master`
<th>Default</th>
<th>Description</th>
</tr>
<tr>
<td>livy.spark.master</td>
<td>local[*]</td>
<td>Spark master uri. ex) spark://masterhost:7077</td>
</tr>
<tr>
<td>zeppelin.livy.url</td>
<td>http://localhost:8998</td>
@ -127,6 +122,8 @@ Example: `spark.master` to `livy.spark.master`
</tr>
</table>
**We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.**
## Adding External libraries
You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version.

14
docs/interpreter/scio.md
View file

@ -52,10 +52,10 @@ In a notebook, to enable the **Scio** interpreter, click the **Gear** icon and s
## Using the Scio Interpreter
In a paragraph, use `$beam.scio` to select the **Scio** interpreter. You can use it much the same way as vanilla Scala REPL and [Scio REPL](https://github.com/spotify/scio/wiki/Scio-REPL). State (like variables, imports, execution etc) is shared among all *Scio* paragraphs. There is a special variable **argz** which holds arguments from Scio interpreter settings. The easiest way to proceed is to create a Scio context via standard `ContextAndArgs`.
In a paragraph, use `%beam.scio` to select the **Scio** interpreter. You can use it much the same way as vanilla Scala REPL and [Scio REPL](https://github.com/spotify/scio/wiki/Scio-REPL). State (like variables, imports, execution etc) is shared among all *Scio* paragraphs. There is a special variable **argz** which holds arguments from Scio interpreter settings. The easiest way to proceed is to create a Scio context via standard `ContextAndArgs`.
```scala
$beam.scio
%beam.scio
val (sc, args) = ContextAndArgs(argz)
```
@ -64,7 +64,7 @@ Use `sc` context the way you would in a regular pipeline/REPL.
Example:
```scala
$beam.scio
%beam.scio
val (sc, args) = ContextAndArgs(argz)
sc.parallelize(Seq("foo", "foo", "bar")).countByValue.closeAndDisplay()
```
@ -108,7 +108,7 @@ There are different helper methods for different objects. You can easily display
#### BigQuery example:
```scala
$beam.scio
%beam.scio
@BigQueryType.fromQuery("""|SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
|FROM [bigquery-samples:airline_ontime_data.flights]
|group by departure_airport
@ -122,7 +122,7 @@ sc.bigQuerySelect(Flights.query).closeAndDisplay(Flights.schema)
#### BigQuery typed example:
```scala
$beam.scio
%beam.scio
@BigQueryType.fromQuery("""|SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
|FROM [bigquery-samples:airline_ontime_data.flights]
|group by departure_airport
@ -136,7 +136,7 @@ sc.typedBigQuery[Flights]().flatMap(_.no_of_delays).mean.closeAndDisplay()
#### Avro example:
```scala
$beam.scio
%beam.scio
import com.spotify.data.ExampleAvro
val (sc, args) = ContextAndArgs(argz)
@ -146,7 +146,7 @@ sc.avroFile[ExampleAvro]("gs://<bucket>/tmp/my.avro").take(10).closeAndDisplay()
#### Avro example with a view schema:
```scala
$beam.scio
%beam.scio
import com.spotify.data.ExampleAvro
import org.apache.avro.Schema

81
docs/manual/interpreterexechooks.md Normal file
View file

@ -0,0 +1,81 @@
---
layout: page
title: "Interpreter Execution Hooks (Experimental)"
description: "Apache Zeppelin allows for users to specify additional code to be executed by an interpreter at pre and post-paragraph code execution."
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Interpreter Execution Hooks (Experimental)
<div id="toc"></div>
## Overview
Apache Zeppelin allows for users to specify additional code to be executed by an interpreter at pre and post-paragraph code execution.
This is primarily useful if you need to run the same set of code for all of the paragraphs within your notebook at specific times.
Currently, this feature is only available for the spark and pyspark interpreters.
To specify your hook code, you may use `z.registerHook()`.
For example, enter the following into one paragraph:
```python
%pyspark
z.registerHook("post_exec", "print 'This code should be executed before the parapgraph code!'")
z.registerHook("pre_exec", "print 'This code should be executed after the paragraph code!'")
```
These calls will not take into effect until the next time you run a paragraph.
In another paragraph, enter
```python
%pyspark
print "This code should be entered into the paragraph by the user!"
```
The output should be:
```
This code should be executed before the paragraph code!
This code should be entered into the paragraph by the user!
This code should be executed after the paragraph code!
```
If you ever need to know the hook code, use `z.getHook()`:
```python
%pyspark
print z.getHook("post_exec")
print 'This code should be executed after the paragraph code!'
```
Any call to `z.registerHook()` will automatically overwrite what was previously registered.
To completely unregister a hook event, use `z.unregisterHook(eventCode)`.
Currently only `"post_exec"` and `"pre_exec"` are valid event codes for the Zeppelin Hook Registry system.
Finally, the hook registry is internally shared by other interpreters in the same group.
This would allow for hook code for one interpreter REPL to be set by another as follows:
```scala
%spark
z.unregisterHook("post_exec", "pyspark")
```
The API is identical for both the spark (scala) and pyspark (python) implementations.
### Caveats
Calls to `z.registerHook("pre_exec", ...)` should be made with care. If there are errors in your specified hook code, this will cause the interpreter REPL to become unable to execute any code pass the pre-execute stage making it impossible for direct calls to `z.unregisterHook()` to take into effect. Current workarounds include calling `z.unregisterHook()` from a different interpreter REPL in the same interpreter group (see above) or manually restarting the interpreter group in the UI.

2
docs/manual/interpreterinstallation.md
View file

@ -108,7 +108,7 @@ You can also install 3rd party interpreters located in the maven repository by u
The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration).
Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/configuration.html).
And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).

46
docs/manual/interpreters.md
View file

@ -82,49 +82,3 @@ interpreter.start()
The above code will start interpreter thread inside your process. Once the interpreter is started you can configure zeppelin to connect to RemoteInterpreter by checking **Connect to existing process** checkbox and then provide **Host** and **Port** on which interpreter process is listening as shown in the image below:
<img src="../assets/themes/zeppelin/img/screenshots/existing_interpreter.png" width="450px">
## (Experimental) Interpreter Execution Hooks
Zeppelin allows for users to specify additional code to be executed by an interpreter at pre and post-paragraph code execution. This is primarily useful if you need to run the same set of code for all of the paragraphs within your notebook at specific times. Currently, this feature is only available for the spark and pyspark interpreters. To specify your hook code, you may use '`z.registerHook()`. For example, enter the following into one paragraph:
```python
%pyspark
z.registerHook("post_exec", "print 'This code should be executed before the parapgraph code!'")
z.registerHook("pre_exec", "print 'This code should be executed after the paragraph code!'")
```
These calls will not take into effect until the next time you run a paragraph. In another paragraph, enter
```python
%pyspark
print "This code should be entered into the paragraph by the user!"
```
The output should be:
```
This code should be executed before the paragraph code!
This code should be entered into the paragraph by the user!
This code should be executed after the paragraph code!
```
If you ever need to know the hook code, use `z.getHook()`:
```python
%pyspark
print z.getHook("post_exec")
```
```
print 'This code should be executed after the paragraph code!'
```
Any call to `z.registerHook()` will automatically overwrite what was previously registered. To completely unregister a hook event, use `z.unregisterHook(eventCode)`. Currently only `"post_exec"` and `"pre_exec"` are valid event codes for the Zeppelin Hook Registry system.
Finally, the hook registry is internally shared by other interpreters in the same group. This would allow for hook code for one interpreter REPL to be set by another as follows:
```scala
%spark
z.unregisterHook("post_exec", "pyspark")
```
The API is identical for both the spark (scala) and pyspark (python) implementations.
### Caveats
Calls to `z.registerHook("pre_exec", ...)` should be made with care. If there are errors in your specified hook code, this will cause the interpreter REPL to become unable to execute any code pass the pre-execute stage making it impossible for direct calls to `z.unregisterHook()` to take into effect. Current workarounds include calling `z.unregisterHook()` from a different interpreter REPL in the same interpreter group (see above) or manually restarting the interpreter group in the UI.

34
docs/quickstart/install_with_flink_and_spark_cluster.md
View file

@ -118,14 +118,16 @@ cd zeppelin
Package Zeppelin.
```
mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.2
mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.3 -Pscala-2.10
```
`-DskipTests` skips build tests- you're not developing (yet), so you don't need to do tests, the clone version *should* build.
`-Pspark-1.6` tells maven to build a Zeppelin with Spark 1.6. This is important because Zeppelin has its own Spark interpreter and the versions must be the same.
`-Dflink.version=1.1.2` tells maven specifically to build Zeppelin with Flink version 1.1.2.
`-Dflink.version=1.1.3` tells maven specifically to build Zeppelin with Flink version 1.1.3.
-`-Pscala-2.10` tells maven to build with Scala v2.10.
**Note:** You may wish to include additional build flags such as `-Ppyspark` or `-Psparkr`. See [the build section of github for more details](https://github.com/apache/zeppelin#build).
@ -162,7 +164,7 @@ Create a new notebook named "Flink Test" and copy and paste the following code.
%flink // let Zeppelin know what interpreter to use.
val text = env.fromElements("In the time of chimpanzees, I was a monkey", // some lines of text to analyze
val text = benv.fromElements("In the time of chimpanzees, I was a monkey", // some lines of text to analyze
"Butane in my veins and I'm out to cut the junkie",
"With the plastic eyeballs, spray paint the vegetables",
"Dog food stalls with the beefcake pantyhose",
@ -252,16 +254,16 @@ Building from source is recommended where possible, for simplicity in this tuto
To download the Flink Binary use `wget`
```bash
wget "http://mirror.cogentco.com/pub/apache/flink/flink-1.0.3/flink-1.0.3-bin-hadoop24-scala_2.10.tgz"
tar -xzvf flink-1.0.3-bin-hadoop24-scala_2.10.tgz
wget "http://mirror.cogentco.com/pub/apache/flink/flink-1.1.3/flink-1.1.3-bin-hadoop24-scala_2.10.tgz"
tar -xzvf flink-1.1.3-bin-hadoop24-scala_2.10.tgz
```
This will download Flink 1.0.3, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `24` to your appropriate version.
This will download Flink 1.1.3, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `24` to your appropriate version.
Start the Flink Cluster.
```bash
flink-1.0.3/bin/start-cluster.sh
flink-1.1.3/bin/start-cluster.sh
```
###### Building From source
@ -270,13 +272,13 @@ If you wish to build Flink from source, the following will be instructive. Note
See the [Flink Installation guide](https://github.com/apache/flink/blob/master/README.md) for more detailed instructions.
Return to the directory where you have been downloading, this tutorial assumes that is `$HOME`. Clone Flink, check out release-1.0, and build.
Return to the directory where you have been downloading, this tutorial assumes that is `$HOME`. Clone Flink, check out release-1.1.3-rc2, and build.
```
cd $HOME
git clone https://github.com/apache/flink.git
cd flink
git checkout release-1.0
git checkout release-1.1.3-rc2
mvn clean install -DskipTests
```
@ -297,8 +299,8 @@ If no task managers are present, restart the Flink cluster with the following co
(if binaries)
```
flink-1.0.3/bin/stop-cluster.sh
flink-1.0.3/bin/start-cluster.sh
flink-1.1.3/bin/stop-cluster.sh
flink-1.1.3/bin/start-cluster.sh
```
@ -320,12 +322,12 @@ Using binaries is also
To download the Spark Binary use `wget`
```bash
wget "http://mirrors.koehn.com/apache/spark/spark-1.6.1/spark-1.6.1-bin-hadoop2.4.tgz"
tar -xzvf spark-1.6.1-bin-hadoop2.4.tgz
mv spark-1.6.1-bin-hadoop4.4 spark
wget "http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz"
tar -xzvf spark-1.6.3-bin-hadoop2.6.tgz
mv spark-1.6.3-bin-hadoop2.6 spark
```
This will download Spark 1.6.1, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `2.4` to your appropriate version.
This will download Spark 1.6.3, compatible with Hadoop 2.6. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `2.6` to your appropriate version.
###### Building From source
@ -335,7 +337,7 @@ See the [Spark Installation](https://github.com/apache/spark/blob/master/README.
Return to the directory where you have been downloading, this tutorial assumes that is $HOME. Clone Spark, check out branch-1.6, and build.
**Note:** Recall, we're only checking out 1.6 because it is the most recent Spark for which a Zeppelin profile exists at
the time of writing. You are free to check out other version, just make sure you build Zeppelin against the correct version of Spark.
the time of writing. You are free to check out other version, just make sure you build Zeppelin against the correct version of Spark. However if you use Spark 2.0, the word count example will need to be changed as Spark 2.0 is not compatible with the following examples.
```

2
docs/security/notebook_authorization.md
View file

@ -46,6 +46,8 @@ If someone who doesn't have **read** permission is trying to access the notebook
<center><img src="../assets/themes/zeppelin/img/docs-img/insufficient_privileges.png"></center>
By default when you create a new note, the owner is the user who create it. And the readers/writers is empty which means it is shared publicly. But if you don't want it to be shared by default. You can set `zeppelin.notebook.public` to be false in `zeppelin-site.xml`.
## How it works
In this section, we will explain the detail about how the notebook authorization works in backend side.

10
docs/security/shiroauthentication.md
View file

@ -41,10 +41,10 @@ cp conf/shiro.ini.template conf/shiro.ini
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
### 3. Secure the Websocket channel
### 2. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
### 4. Start Zeppelin
### 3. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
@ -52,7 +52,7 @@ bin/zeppelin-daemon.sh start (or restart)
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
### 5. Login
### 4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
@ -161,7 +161,7 @@ zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
securityManager.realms = $zeppelinHubRealm
```
> Note: ZeppelinHub is not releated to apache Zeppelin project.
> Note: ZeppelinHub is not releated to Apache Zeppelin project.
## Secure your Zeppelin information (optional)
By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
@ -180,7 +180,7 @@ In this case, only who have `admin` role can see **Interpreter Setting**, **Cred
If you want to grant this permission to other users, you can change **roles[ ]** as you defined at `[users]` section.
<br/>
> **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file. This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).
> **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file.
## Other authentication methods

2
elasticsearch/pom.xml
View file

@ -33,7 +33,7 @@
<name>Zeppelin: Elasticsearch interpreter</name>
<properties>
<elasticsearch.version>2.3.3</elasticsearch.version>
<elasticsearch.version>2.4.3</elasticsearch.version>
<guava.version>18.0</guava.version>
<json-flattener.version>0.1.6</json-flattener.version>
</properties>

1
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -46,7 +46,6 @@ public class FlinkInterpreterTest {
@AfterClass
public static void tearDown() {
flink.close();
flink.destroy();
}
@Test

2
ignite/pom.xml
View file

@ -32,7 +32,7 @@
<name>Zeppelin: Apache Ignite interpreter</name>
<properties>
<ignite.version>1.7.0</ignite.version>
<ignite.version>1.8.0</ignite.version>
</properties>
<dependencies>

17
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -76,23 +76,6 @@ public class IgniteInterpreter extends Interpreter {
static final String IGNITE_CFG_URL = "ignite.config.url";
static {
Interpreter.register(
"ignite",
"ignite",
IgniteInterpreter.class.getName(),
true,
new InterpreterPropertyBuilder()
.add(IGNITE_ADDRESSES, "127.0.0.1:47500..47509",
"Coma separated list of addresses "
+ "(e.g. 127.0.0.1:47500 or 127.0.0.1:47500..47509)")
.add(IGNITE_CLIENT_MODE, "true", "Client mode. true or false")
.add(IGNITE_CFG_URL, "", "Configuration URL. Overrides all other settings.")
.add(IGNITE_PEER_CLASS_LOADING_ENABLED, "true",
"Peer class loading enabled. true or false")
.build());
}
private Logger logger = LoggerFactory.getLogger(IgniteInterpreter.class);
private Ignite ignite;
private ByteArrayOutputStream out;

11
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java
View file

@ -57,17 +57,6 @@ public class IgniteSqlInterpreter extends Interpreter {
static final String IGNITE_JDBC_URL = "ignite.jdbc.url";
static {
Interpreter.register(
"ignitesql",
"ignite",
IgniteSqlInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(IGNITE_JDBC_URL,
"jdbc:ignite:cfg://default-ignite-jdbc.xml", "Ignite JDBC connection URL.")
.build());
}
private Logger logger = LoggerFactory.getLogger(IgniteSqlInterpreter.class);
private Connection conn;

46
ignite/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,46 @@
[
{
"group": "ignite",
"name": "ignite",
"className": "org.apache.zeppelin.ignite.IgniteInterpreter",
"properties": {
"ignite.addresses": {
"envName": null,
"propertyName": "ignite.addresses",
"defaultValue": "127.0.0.1:47500..47509",
"description": "Comma separated list of addresses (e.g. 127.0.0.1:47500 or 127.0.0.1:47500..47509)"
},
"ignite.clientMode": {
"envName": null,
"propertyName": "ignite.clientMode",
"defaultValue": "true",
"description": "Client mode. true or false"
},
"ignite.config.url": {
"envName": null,
"propertyName": "ignite.config.url",
"defaultValue": "",
"description": "Configuration URL. Overrides all other settings."
},
"ignite.peerClassLoadingEnabled": {
"envName": null,
"propertyName": "ignite.peerClassLoadingEnabled",
"defaultValue": "true",
"description": "Peer class loading enabled. True or false"
}
}
},
{
"group": "ignite",
"name": "ignitesql",
"className": "org.apache.zeppelin.ignite.IgniteSqlInterpreter",
"properties": {
"ignite.jdbc.url": {
"envName": null,
"propertyName": "ignite.jdbc.url",
"defaultValue": "jdbc:ignite:cfg://default-ignite-jdbc.xml",
"description": "Ignite JDBC connection URL."
}
}
}
]

1
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -64,6 +64,7 @@ public class IgniteInterpreterTest {
props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, "jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml");
props.setProperty(IgniteInterpreter.IGNITE_CLIENT_MODE, "false");
props.setProperty(IgniteInterpreter.IGNITE_PEER_CLASS_LOADING_ENABLED, "false");
props.setProperty(IgniteInterpreter.IGNITE_ADDRESSES, HOST);
intp = new IgniteInterpreter(props);
intp.open();

118
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -80,7 +80,7 @@ public class JDBCInterpreter extends Interpreter {
static final String JDBC_DEFAULT_PASSWORD_KEY = "default.password";
static final String COMMON_KEY = "common";
static final String MAX_LINE_KEY = "max_count";
static final String MAX_LINE_DEFAULT = "1000";
static final int MAX_LINE_DEFAULT = 1000;
static final String DEFAULT_KEY = "default";
static final String DRIVER_KEY = "driver";
@ -121,12 +121,14 @@ public class JDBCInterpreter extends Interpreter {
};
private static final List<InterpreterCompletion> NO_COMPLETION = new ArrayList<>();
private int maxLineResults;
public JDBCInterpreter(Properties property) {
super(property);
jdbcUserConfigurationsMap = new HashMap<>();
propertyKeySqlCompleterMap = new HashMap<>();
basePropretiesMap = new HashMap<>();
maxLineResults = MAX_LINE_DEFAULT;
}
public HashMap<String, Properties> getPropertiesMap() {
@ -146,9 +148,9 @@ public class JDBCInterpreter extends Interpreter {
prefixProperties = basePropretiesMap.get(keyValue[0]);
} else {
prefixProperties = new Properties();
basePropretiesMap.put(keyValue[0], prefixProperties);
basePropretiesMap.put(keyValue[0].trim(), prefixProperties);
}
prefixProperties.put(keyValue[1], property.getProperty(propertyKey));
prefixProperties.put(keyValue[1].trim(), property.getProperty(propertyKey));
}
}
@ -175,6 +177,14 @@ public class JDBCInterpreter extends Interpreter {
for (String propertyKey : basePropretiesMap.keySet()) {
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
}
setMaxLineResults();
}
private void setMaxLineResults() {
if (basePropretiesMap.containsKey(COMMON_KEY) &&
basePropretiesMap.get(COMMON_KEY).containsKey(MAX_LINE_KEY)) {
maxLineResults = Integer.valueOf(basePropretiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY));
}
}
private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
@ -389,6 +399,50 @@ public class JDBCInterpreter extends Interpreter {
return connection;
}
private String getResults(ResultSet resultSet, boolean isTableType)
throws SQLException {
ResultSetMetaData md = resultSet.getMetaData();
StringBuilder msg;
if (isTableType) {
msg = new StringBuilder(TABLE_MAGIC_TAG);
} else {
msg = new StringBuilder();
}
for (int i = 1; i < md.getColumnCount() + 1; i++) {
if (i > 1) {
msg.append(TAB);
}
msg.append(replaceReservedChars(md.getColumnName(i)));
}
msg.append(NEWLINE);
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
Object resultObject;
String resultValue;
resultObject = resultSet.getObject(i);
if (resultObject == null) {
resultValue = "null";
} else {
resultValue = resultSet.getString(i);
}
msg.append(replaceReservedChars(resultValue));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
}
msg.append(NEWLINE);
displayRowCount++;
}
return msg.toString();
}
private boolean isDDLCommand(int updatedCount, int columnCount) throws SQLException {
return updatedCount < 0 && columnCount <= 0 ? true : false;
}
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
Connection connection;
@ -398,6 +452,7 @@ public class JDBCInterpreter extends Interpreter {
String user = interpreterContext.getAuthenticationInfo().getUser();
try {
String results = null;
connection = getConnection(propertyKey, interpreterContext);
if (connection == null) {
@ -409,61 +464,27 @@ public class JDBCInterpreter extends Interpreter {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
StringBuilder msg = null;
boolean isTableType = false;
if (containsIgnoreCase(sql, EXPLAIN_PREDICATE)) {
msg = new StringBuilder();
} else {
msg = new StringBuilder(TABLE_MAGIC_TAG);
isTableType = true;
}
try {
getJDBCConfiguration(user).saveStatement(paragraphId, statement);
boolean isResultSetAvailable = statement.execute(sql);
if (isResultSetAvailable) {
resultSet = statement.getResultSet();
ResultSetMetaData md = resultSet.getMetaData();
for (int i = 1; i < md.getColumnCount() + 1; i++) {
if (i > 1) {
msg.append(TAB);
}
msg.append(replaceReservedChars(isTableType, md.getColumnName(i)));
}
msg.append(NEWLINE);
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
Object resultObject;
String resultValue;
resultObject = resultSet.getObject(i);
if (resultObject == null) {
resultValue = "null";
} else {
resultValue = resultSet.getString(i);
}
msg.append(replaceReservedChars(isTableType, resultValue));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
}
msg.append(NEWLINE);
displayRowCount++;
// Regards that the command is DDL.
if (isDDLCommand(statement.getUpdateCount(), resultSet.getMetaData().getColumnCount())) {
results = "Query executed successfully.";
} else {
results = getResults(resultSet, !containsIgnoreCase(sql, EXPLAIN_PREDICATE));
}
} else {
// Response contains either an update count or there are no results.
int updateCount = statement.getUpdateCount();
msg.append(UPDATE_COUNT_HEADER).append(NEWLINE);
msg.append(updateCount).append(NEWLINE);
results = "Query executed successfully. Affected rows : " + updateCount;
}
//In case user ran an insert/update/upsert statement
if (connection.getAutoCommit() != true) connection.commit();
} finally {
if (resultSet != null) {
try {
@ -482,7 +503,7 @@ public class JDBCInterpreter extends Interpreter {
}
getJDBCConfiguration(user).removeStatement(paragraphId);
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
return new InterpreterResult(Code.SUCCESS, results);
} catch (Exception e) {
logger.error("Cannot run " + sql, e);
@ -504,11 +525,11 @@ public class JDBCInterpreter extends Interpreter {
/**
* For %table response replace Tab and Newline characters from the content.
*/
private String replaceReservedChars(boolean isTableResponseType, String str) {
private String replaceReservedChars(String str) {
if (str == null) {
return EMPTY_COLUMN_VALUE;
}
return (!isTableResponseType) ? str : str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE);
return str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE);
}
@Override
@ -589,8 +610,7 @@ public class JDBCInterpreter extends Interpreter {
}
public int getMaxResult() {
return Integer.valueOf(
basePropretiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
return maxLineResults;
}
boolean isConcurrentExecution() {

27
kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java
View file

@ -15,7 +15,6 @@
* limitations under the License.
*/
package org.apache.zeppelin.kylin;
import org.apache.commons.codec.binary.Base64;
@ -41,7 +40,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Kylin interpreter for Zeppelin. (http://kylin.io)
* Kylin interpreter for Zeppelin. (http://kylin.apache.org)
*/
public class KylinInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(KylinInterpreter.class);
@ -59,6 +58,7 @@ public class KylinInterpreter extends Interpreter {
public KylinInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
@ -100,8 +100,9 @@ public class KylinInterpreter extends Interpreter {
}
public HttpResponse prepareRequest(String sql) throws IOException {
String KYLIN_PROJECT = getProperty(KYLIN_QUERY_PROJECT);
logger.info("project:" + KYLIN_PROJECT);
String kylinProject = getProject(KYLIN_QUERY_PROJECT);
logger.info("project:" + kylinProject);
logger.info("sql:" + sql);
logger.info("acceptPartial:" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL));
logger.info("limit:" + getProperty(KYLIN_QUERY_LIMIT));
@ -109,7 +110,7 @@ public class KylinInterpreter extends Interpreter {
byte[] encodeBytes = Base64.encodeBase64(new String(getProperty(KYLIN_USERNAME)
+ ":" + getProperty(KYLIN_PASSWORD)).getBytes("UTF-8"));
String postContent = new String("{\"project\":" + "\"" + KYLIN_PROJECT + "\""
String postContent = new String("{\"project\":" + "\"" + kylinProject + "\""
+ "," + "\"sql\":" + "\"" + sql + "\""
+ "," + "\"acceptPartial\":" + "\"" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL) + "\""
+ "," + "\"offset\":" + "\"" + getProperty(KYLIN_QUERY_OFFSET) + "\""
@ -130,6 +131,22 @@ public class KylinInterpreter extends Interpreter {
return httpClient.execute(postRequest);
}
public String getProject(String cmd) {
boolean firstLineIndex = cmd.startsWith("(");
if (firstLineIndex) {
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1) {
return cmd.substring(configStartIndex + 1, configLastIndex);
} else {
return getProperty(KYLIN_QUERY_PROJECT);
}
} else {
return getProperty(KYLIN_QUERY_PROJECT);
}
}
private InterpreterResult executeQuery(String sql) throws IOException {
HttpResponse response = prepareRequest(sql);

14
kylin/src/main/resources/interpreter-setting.json
View file

@ -14,37 +14,37 @@
"envName": null,
"propertyName": "kylin.api.user",
"defaultValue": "ADMIN",
"description": "username for kylin user"
"description": "Kylin username"
},
"kylin.api.password": {
"envName": null,
"propertyName": "kylin.api.password",
"defaultValue": "KYLIN",
"description": "password for kylin user"
"description": "Kylin password"
},
"kylin.query.project": {
"envName": null,
"propertyName": "kylin.query.project",
"defaultValue": "default",
"description": "kylin project name"
"defaultValue": "learn_kylin",
"description": "Default Kylin project name"
},
"kylin.query.offset": {
"envName": null,
"propertyName": "kylin.query.offset",
"defaultValue": "0",
"description": "kylin query offset"
"description": "Kylin query offset"
},
"kylin.query.limit": {
"envName": null,
"propertyName": "kylin.query.limit",
"defaultValue": "5000",
"description": "kylin query limit"
"description": "Kylin query limit"
},
"kylin.query.ispartial": {
"envName": null,
"propertyName": "kylin.query.ispartial",
"defaultValue": "true",
"description": "The kylin query partial flag"
"description": "Kylin query partial flag, deprecated"
}
},
"editor": {

27
kylin/src/test/java/KylinInterpreterTest.java
View file

@ -47,13 +47,36 @@ public class KylinInterpreterTest {
}
@Test
public void test(){
KylinInterpreter t = new MockKylinInterpreter(kylinProperties);
public void testWithDefault(){
KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
InterpreterResult result = t.interpret(
"select a.date,sum(b.measure) as measure from kylin_fact_table a " +
"inner join kylin_lookup_table b on a.date=b.date group by a.date", null);
assertEquals("default", t.getProject("select a.date,sum(b.measure) as measure " +
"from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
assertEquals(InterpreterResult.Type.TABLE,result.message().get(0).getType());
}
@Test
public void testWithProject(){
KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
assertEquals("project2", t.getProject("(project2)\n select a.date,sum(b.measure) as measure " +
"from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
assertEquals("", t.getProject("()\n select a.date,sum(b.measure) as measure " +
"from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date group by a.date"));
}
private Properties getDefaultProperties(){
Properties prop = new Properties();
prop.put("kylin.api.username", "ADMIN");
prop.put("kylin.api.password", "KYLIN");
prop.put("kylin.api.url", "http://<host>:<port>/kylin/api/query");
prop.put("kylin.query.project", "default");
prop.put("kylin.query.offset", "0");
prop.put("kylin.query.limit", "5000");
prop.put("kylin.query.ispartial", "true");
return prop;
}
}
class MockKylinInterpreter extends KylinInterpreter {

17
livy/README.md Normal file
View file

@ -0,0 +1,17 @@
# Overview
Livy interpreter for Apache Zeppelin
# Prerequisities
You can follow the instructions at [Livy Quick Start](http://livy.io/quickstart.html) to set up livy.
# Run Integration Tests
You can add integration test to [LivyInterpreter.java](https://github.com/apache/zeppelin/blob/master/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java)
Either you can run the integration test on travis where enviroment will be setup or you can run it in local. You need to download livy-0.2 and spark-1.5.2 to local, then use the following
script to run the integration test.
```bash
#!/usr/bin/env bash
export LIVY_HOME=<path_of_livy_0.2.0>
export SPARK_HOME=<path_of_spark-1.5.2>
mvn clean verify -pl livy -DfailIfNoTests=false -DskipRat
```

449
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -24,7 +27,17 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
@ -33,76 +46,83 @@ import java.util.Properties;
public abstract class BaseLivyInterprereter extends Interpreter {
protected static final Logger LOGGER = LoggerFactory.getLogger(BaseLivyInterprereter.class);
private static Gson gson = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create();
// -1 means session is not created yet, valid sessionId start from 0
protected int sessionId = -1;
protected String appId;
protected String webUIAddress;
protected SessionInfo sessionInfo;
private String livyURL;
private long sessionCreationTimeout;
protected boolean displayAppInfo;
protected LivyOutputStream out;
protected LivyHelper livyHelper;
public BaseLivyInterprereter(Properties property) {
super(property);
this.out = new LivyOutputStream();
this.livyHelper = new LivyHelper(property);
this.livyURL = property.getProperty("zeppelin.livy.url");
this.sessionCreationTimeout = Long.parseLong(
property.getProperty("zeppelin.livy.create.session.timeout", 120 + ""));
}
public abstract String getSessionKind();
@Override
public void open() {
// TODO(zjffdu) move session creation here.
try {
initLivySession();
} catch (LivyException e) {
String msg = "Fail to create session, please check livy interpreter log and " +
"livy server log";
LOGGER.error(msg);
throw new RuntimeException(msg, e);
}
}
@Override
public void close() {
if (sessionId != -1) {
livyHelper.closeSession(sessionId);
// reset sessionId to -1
sessionId = -1;
if (sessionInfo != null) {
closeSession(sessionInfo.id);
// reset sessionInfo to null so that we won't close it twice.
sessionInfo = null;
}
}
protected void createSession(InterpreterContext context) throws Exception {
sessionId = livyHelper.createSession(context, getSessionKind());
protected void initLivySession() throws LivyException {
this.sessionInfo = createSession(getUserName(), getSessionKind());
if (displayAppInfo) {
this.appId = extractStatementResult(
livyHelper.interpret("sc.applicationId", context, sessionId).message().get(0).getData());
livyHelper.interpret(
"val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
context, sessionId);
this.webUIAddress = extractStatementResult(
livyHelper.interpret(
"webui.getClass.getMethod(\"appUIAddress\").invoke(webui)",
context, sessionId).message().get(0).getData());
LOGGER.info("Create livy session with sessionId: {}, appId: {}, webUI: {}",
sessionId, appId, webUIAddress);
if (sessionInfo.appId == null) {
// livy 0.2 don't return appId and sparkUiUrl in response so that we need to get it
// explicitly by ourselves.
sessionInfo.appId = extractStatementResult(
interpret("sc.applicationId", false).message()
.get(0).getData());
}
interpret(
"val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get", false);
if (StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
sessionInfo.webUIAddress = extractStatementResult(
interpret(
"webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", false)
.message().get(0).getData());
} else {
sessionInfo.webUIAddress = sessionInfo.appInfo.get("sparkUiUrl");
}
LOGGER.info("Create livy session successfully with sessionId: {}, appId: {}, webUI: {}",
sessionInfo.id, sessionInfo.appId, sessionInfo.webUIAddress);
}
}
public SessionInfo getSessionInfo() {
return sessionInfo;
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
try {
// add synchronized, because LivySparkSQLInterperter will use ParallelScheduler
synchronized (this) {
if (sessionId == -1) {
try {
createSession(context);
} catch (Exception e) {
LOGGER.error("Exception while creating livy session", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
}
if (StringUtils.isEmpty(st)) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
if (StringUtils.isEmpty(st)) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpretInput(st, context, sessionId, out,
appId, webUIAddress, displayAppInfo);
} catch (Exception e) {
LOGGER.error("Exception in LivyInterpreter.", e);
try {
return interpret(st, this.displayAppInfo);
} catch (LivyException e) {
LOGGER.error("Fail to interpret:" + st, e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
@ -116,7 +136,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
* @param result
* @return
*/
private static String extractStatementResult(String result) {
private String extractStatementResult(String result) {
int pos = -1;
if ((pos = result.indexOf("=")) >= 0) {
return result.substring(pos + 1).trim();
@ -128,7 +148,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
//TODO(zjffdu). Use livy cancel api which is available in livy 0.3
}
@Override
@ -140,4 +160,339 @@ public abstract class BaseLivyInterprereter extends Interpreter {
public int getProgress(InterpreterContext context) {
return 0;
}
private SessionInfo createSession(String user, String kind)
throws LivyException {
try {
Map<String, String> conf = new HashMap<>();
for (Map.Entry<Object, Object> entry : property.entrySet()) {
if (entry.getKey().toString().startsWith("livy.spark.") &&
!entry.getValue().toString().isEmpty())
conf.put(entry.getKey().toString().substring(5), entry.getValue().toString());
}
CreateSessionRequest request = new CreateSessionRequest(kind, user, conf);
SessionInfo sessionInfo = SessionInfo.fromJson(
callRestAPI("/sessions", "POST", request.toJson()));
long start = System.currentTimeMillis();
// pull the session status until it is idle or timeout
while (!sessionInfo.isReady()) {
LOGGER.info("Session {} is in state {}, appId {}", sessionInfo.id, sessionInfo.state,
sessionInfo.appId);
if (sessionInfo.isFinished()) {
String msg = "Session " + sessionInfo.id + " is finished, appId: " + sessionInfo.appId
+ ", log: " + sessionInfo.log;
LOGGER.error(msg);
throw new LivyException(msg);
}
if ((System.currentTimeMillis() - start) / 1000 > sessionCreationTimeout) {
String msg = "The creation of session " + sessionInfo.id + " is timeout within "
+ sessionCreationTimeout + " seconds, appId: " + sessionInfo.appId
+ ", log: " + sessionInfo.log;
LOGGER.error(msg);
throw new LivyException(msg);
}
Thread.sleep(1000);
sessionInfo = getSessionInfo(sessionInfo.id);
}
return sessionInfo;
} catch (Exception e) {
LOGGER.error("Error when creating livy session for user " + user, e);
throw new LivyException(e);
}
}
private SessionInfo getSessionInfo(int sessionId) throws LivyException {
return SessionInfo.fromJson(callRestAPI("/sessions/" + sessionId, "GET"));
}
public InterpreterResult interpret(String code, boolean displayAppInfo)
throws LivyException {
StatementInfo stmtInfo = executeStatement(new ExecuteRequest(code));
// pull the statement status
while (!stmtInfo.isAvailable()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOGGER.error("InterruptedException when pulling statement status.", e);
throw new LivyException(e);
}
stmtInfo = getStatementInfo(stmtInfo.id);
}
return getResultFromStatementInfo(stmtInfo, displayAppInfo);
}
private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo,
boolean displayAppInfo) {
if (stmtInfo.output.isError()) {
return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue);
} else {
//TODO(zjffdu) support other types of data (like json, image and etc)
String result = stmtInfo.output.data.plain_text;
// check table magic result first
if (stmtInfo.output.data.application_livy_table_json != null) {
StringBuilder outputBuilder = new StringBuilder();
boolean notFirstColumn = false;
for (Map header : stmtInfo.output.data.application_livy_table_json.headers) {
if (notFirstColumn) {
outputBuilder.append("\t");
}
outputBuilder.append(header.get("name"));
notFirstColumn = true;
}
outputBuilder.append("\n");
for (List<Object> row : stmtInfo.output.data.application_livy_table_json.records) {
outputBuilder.append(StringUtils.join(row, "\t"));
outputBuilder.append("\n");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TABLE, outputBuilder.toString());
} else if (stmtInfo.output.data.image_png != null) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png);
} else if (result != null) {
result = result.trim();
if (result.startsWith("<link")
|| result.startsWith("<script")
|| result.startsWith("<style")
|| result.startsWith("<div")) {
result = "%html " + result;
}
}
if (displayAppInfo) {
//TODO(zjffdu), use multiple InterpreterResult to display appInfo
StringBuilder outputBuilder = new StringBuilder();
outputBuilder.append("%angular ");
outputBuilder.append("<pre><code>");
outputBuilder.append(result);
outputBuilder.append("</code></pre>");
outputBuilder.append("<hr/>");
outputBuilder.append("Spark Application Id:" + sessionInfo.appId + "<br/>");
outputBuilder.append("Spark WebUI: <a href=" + sessionInfo.webUIAddress + ">"
+ sessionInfo.webUIAddress + "</a>");
return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputBuilder.toString());
} else {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, result);
}
}
}
private StatementInfo executeStatement(ExecuteRequest executeRequest)
throws LivyException {
return StatementInfo.fromJson(callRestAPI("/sessions/" + sessionInfo.id + "/statements", "POST",
executeRequest.toJson()));
}
private StatementInfo getStatementInfo(int statementId)
throws LivyException {
return StatementInfo.fromJson(
callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId, "GET"));
}
private RestTemplate getRestTemplate() {
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
return new KerberosRestTemplate(keytabLocation, principal);
}
return new RestTemplate();
}
private String callRestAPI(String targetURL, String method) throws LivyException {
return callRestAPI(targetURL, method, "");
}
private String callRestAPI(String targetURL, String method, String jsonData)
throws LivyException {
targetURL = livyURL + targetURL;
LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData);
RestTemplate restTemplate = getRestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("Content-Type", "application/json");
headers.add("X-Requested-By", "zeppelin");
ResponseEntity<String> response = null;
try {
if (method.equals("POST")) {
HttpEntity<String> entity = new HttpEntity<>(jsonData, headers);
response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class);
} else if (method.equals("GET")) {
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class);
} else if (method.equals("DELETE")) {
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class);
}
} catch (HttpClientErrorException e) {
response = new ResponseEntity(e.getResponseBodyAsString(), e.getStatusCode());
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), e.getResponseBodyAsString()));
}
if (response == null) {
throw new LivyException("No http response returned");
}
LOGGER.debug("Get response, StatusCode: {}, responseBody: {}", response.getStatusCode(),
response.getBody());
if (response.getStatusCode().value() == 200
|| response.getStatusCode().value() == 201
|| response.getStatusCode().value() == 404) {
String responseBody = response.getBody();
if (responseBody.matches("Session '\\d+' not found.")) {
throw new SessionNotFoundException(responseBody);
} else {
return responseBody;
}
} else {
String responseString = response.getBody();
if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
return responseString;
}
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), responseString));
throw new LivyException(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), responseString));
}
}
private void closeSession(int sessionId) {
try {
callRestAPI("/sessions/" + sessionId, "DELETE");
} catch (Exception e) {
LOGGER.error(String.format("Error closing session for user with session ID: %s",
sessionId), e);
}
}
/*
* We create these POJO here to accommodate livy 0.3 which is not released yet. livy rest api has
* some changes from version to version. So we create these POJO in zeppelin side to accommodate
* incompatibility between versions. Later, when livy become more stable, we could just depend on
* livy client jar.
*/
private static class CreateSessionRequest {
public final String kind;
@SerializedName("proxyUser")
public final String user;
public final Map<String, String> conf;
public CreateSessionRequest(String kind, String user, Map<String, String> conf) {
this.kind = kind;
this.user = user;
this.conf = conf;
}
public String toJson() {
return gson.toJson(this);
}
}
/**
*
*/
public static class SessionInfo {
public final int id;
public String appId;
public String webUIAddress;
public final String owner;
public final String proxyUser;
public final String state;
public final String kind;
public final Map<String, String> appInfo;
public final List<String> log;
public SessionInfo(int id, String appId, String owner, String proxyUser, String state,
String kind, Map<String, String> appInfo, List<String> log) {
this.id = id;
this.appId = appId;
this.owner = owner;
this.proxyUser = proxyUser;
this.state = state;
this.kind = kind;
this.appInfo = appInfo;
this.log = log;
}
public boolean isReady() {
return state.equals("idle");
}
public boolean isFinished() {
return state.equals("error") || state.equals("dead") || state.equals("success");
}
public static SessionInfo fromJson(String json) {
return gson.fromJson(json, SessionInfo.class);
}
}
private static class ExecuteRequest {
public final String code;
public ExecuteRequest(String code) {
this.code = code;
}
public String toJson() {
return gson.toJson(this);
}
}
private static class StatementInfo {
public Integer id;
public String state;
public StatementOutput output;
public StatementInfo() {
}
public static StatementInfo fromJson(String json) {
return gson.fromJson(json, StatementInfo.class);
}
public boolean isAvailable() {
return state.equals("available");
}
private static class StatementOutput {
public String status;
public String execution_count;
public Data data;
public String ename;
public String evalue;
public Object traceback;
public TableMagic tableMagic;
public boolean isError() {
return status.equals("error");
}
public String toJson() {
return gson.toJson(this);
}
private static class Data {
@SerializedName("text/plain")
public String plain_text;
@SerializedName("image/png")
public String image_png;
@SerializedName("application/json")
public String application_json;
@SerializedName("application/vnd.livy.table.v1+json")
public TableMagic application_livy_table_json;
}
private static class TableMagic {
@SerializedName("headers")
List<Map> headers;
@SerializedName("data")
List<List> records;
}
}
}
}

43
livy/src/main/java/org/apache/zeppelin/livy/LivyException.java Normal file
View file

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
/**
* Livy api related exception
*/
public class LivyException extends Exception {
public LivyException() {
}
public LivyException(String message) {
super(message);
}
public LivyException(String message, Throwable cause) {
super(message, cause);
}
public LivyException(Throwable cause) {
super(cause);
}
public LivyException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

406
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
View file

@ -1,406 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import java.nio.charset.Charset;
import java.util.*;
import java.util.Map.Entry;
/***
* Livy helper class
*/
public class LivyHelper {
Logger LOGGER = LoggerFactory.getLogger(LivyHelper.class);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
HashMap<String, Object> paragraphHttpMap = new HashMap<>();
Properties property;
LivyHelper(Properties property) {
this.property = property;
}
public Integer createSession(InterpreterContext context, String kind) throws Exception {
try {
Map<String, String> conf = new HashMap<>();
Iterator<Entry<Object, Object>> it = property.entrySet().iterator();
while (it.hasNext()) {
Entry<Object, Object> pair = it.next();
if (pair.getKey().toString().startsWith("livy.spark.") &&
!pair.getValue().toString().isEmpty())
conf.put(pair.getKey().toString().substring(5), pair.getValue().toString());
}
String confData = gson.toJson(conf);
String user = context.getAuthenticationInfo().getUser();
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions", "POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"conf\": " + confData + ", " +
"\"proxyUser\": " + (StringUtils.isEmpty(user) ? null : "\"" + user + "\"") +
"}",
context.getParagraphId()
);
Map jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
Integer sessionId = ((Double) jsonMap.get("id")).intValue();
if (!jsonMap.get("state").equals("idle")) {
Integer retryCount = 60;
try {
retryCount = Integer.valueOf(
property.getProperty("zeppelin.livy.create.session.retries"));
} catch (Exception e) {
LOGGER.info("zeppelin.livy.create.session.retries property is not configured." +
" Using default retry count.");
}
while (retryCount >= 0) {
LOGGER.error(String.format("sessionId:%s state is %s",
jsonMap.get("id"), jsonMap.get("state")));
Thread.sleep(1000);
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId, "GET", null, context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
if (jsonMap.get("state").equals("idle")) {
break;
} else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId + "/log",
"GET", null,
context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
String logs = StringUtils.join((ArrayList<String>) jsonMap.get("log"), '\n');
LOGGER.error(String.format("Cannot start %s.\n%s", kind, logs));
throw new Exception(String.format("Cannot start %s.\n%s", kind, logs));
}
retryCount--;
}
if (retryCount <= 0) {
LOGGER.error("Error getting session for user within the configured number of retries.");
throw new Exception(String.format("Cannot start %s.", kind));
}
}
return sessionId;
} catch (Exception e) {
LOGGER.error("Error getting session for user", e);
throw e;
}
}
public InterpreterResult interpretInput(String stringLines,
final InterpreterContext context,
int sessionId,
LivyOutputStream out,
String appId,
String webUI,
boolean displayAppInfo) {
try {
out.setInterpreterOutput(context.out);
context.out.clear();
String incomplete = "";
boolean inComment = false;
String[] lines = stringLines.split("\n");
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
Code r = null;
StringBuilder outputBuilder = new StringBuilder();
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
//for spark
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
boolean continuation = false;
if (nextLine.isEmpty()
|| nextLine.startsWith("//") // skip empty line or comment
|| nextLine.startsWith("}")
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
continuation = true;
} else if (!inComment && nextLine.startsWith("/*")) {
inComment = true;
continuation = true;
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
inComment = false;
continuation = true;
} else if (nextLine.length() > 1
&& nextLine.charAt(0) == '.'
&& nextLine.charAt(1) != '.' // ".."
&& nextLine.charAt(1) != '/') { // "./"
continuation = true;
} else if (inComment) {
continuation = true;
}
if (continuation) {
incomplete += s + "\n";
continue;
}
}
InterpreterResult res;
try {
res = interpret(incomplete + s, context, sessionId);
} catch (Exception e) {
LOGGER.error("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = res.code();
if (r == Code.ERROR) {
out.setInterpreterOutput(null);
return res;
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
outputBuilder.append(res.message() + "\n");
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
out.setInterpreterOutput(null);
return new InterpreterResult(r, "Incomplete expression");
} else {
if (displayAppInfo) {
out.write("%angular ");
out.write("<pre><code>");
out.write(outputBuilder.toString());
out.write("</code></pre>");
out.write("<hr/>");
out.write("Spark Application Id:" + appId + "<br/>");
out.write("Spark WebUI: <a href=" + webUI + ">" + webUI + "</a>");
} else {
out.write(outputBuilder.toString());
}
out.setInterpreterOutput(null);
return new InterpreterResult(Code.SUCCESS);
}
} catch (Exception e) {
LOGGER.error("error in interpretInput", e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
public InterpreterResult interpret(String stringLines,
final InterpreterContext context,
int sessionId)
throws Exception {
if (stringLines.trim().equals("")) {
return new InterpreterResult(Code.SUCCESS, "");
}
Map jsonMap = executeCommand(stringLines, context, sessionId);
Integer id = ((Double) jsonMap.get("id")).intValue();
InterpreterResult res = getResultFromMap(jsonMap);
if (res != null) {
return res;
}
while (true) {
Thread.sleep(1000);
if (paragraphHttpMap.get(context.getParagraphId()) == null) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
jsonMap = getStatusById(context, sessionId, id);
InterpreterResult interpreterResult = getResultFromMap(jsonMap);
if (interpreterResult != null) {
return interpreterResult;
}
}
}
private InterpreterResult getResultFromMap(Map jsonMap) {
if (jsonMap.get("state").equals("available")) {
if (((Map) jsonMap.get("output")).get("status").equals("error")) {
StringBuilder errorMessage = new StringBuilder((String) ((Map) jsonMap
.get("output")).get("evalue"));
if (errorMessage.toString().equals("incomplete statement")
|| errorMessage.toString().contains("EOF")) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
String traceback = gson.toJson(((Map) jsonMap.get("output")).get("traceback"));
if (!traceback.equals("[]")) {
errorMessage
.append("\n")
.append("traceback: \n")
.append(traceback);
}
return new InterpreterResult(Code.ERROR, errorMessage.toString());
}
if (((Map) jsonMap.get("output")).get("status").equals("ok")) {
String result = (String) ((Map) ((Map) jsonMap.get("output"))
.get("data")).get("text/plain");
if (result != null) {
result = result.trim();
if (result.startsWith("<link")
|| result.startsWith("<script")
|| result.startsWith("<style")
|| result.startsWith("<div")) {
result = "%html " + result;
}
}
return new InterpreterResult(Code.SUCCESS, result);
}
}
return null;
}
private Map executeCommand(String lines, InterpreterContext context, int sessionId)
throws Exception {
String json = executeHTTP(property.get("zeppelin.livy.url") + "/sessions/"
+ sessionId + "/statements",
"POST",
"{\"code\": \"" + StringEscapeUtils.escapeJson(lines) + "\"}",
context.getParagraphId());
if (json.matches("^(\")?Session (\'[0-9]\' )?not found(.?\"?)$")) {
throw new Exception("Exception: Session not found, Livy server would have restarted, " +
"or lost session.");
}
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error executeCommand", e);
throw e;
}
}
private Map getStatusById(InterpreterContext context,
int sessionId, Integer id) throws Exception {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
+ sessionId
+ "/statements/" + id,
"GET", null, context.getParagraphId());
LOGGER.debug("statement {} response: {}", id, json);
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error getStatusById", e);
throw e;
}
}
private RestTemplate getRestTemplate() {
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
return new KerberosRestTemplate(keytabLocation, principal);
}
return new RestTemplate();
}
protected String executeHTTP(String targetURL, String method, String jsonData, String paragraphId)
throws Exception {
LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData);
RestTemplate restTemplate = getRestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("Content-Type", "application/json");
headers.add("X-Requested-By", "zeppelin");
ResponseEntity<String> response = null;
try {
if (method.equals("POST")) {
HttpEntity<String> entity = new HttpEntity<>(jsonData, headers);
response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("GET")) {
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("DELETE")) {
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class);
}
} catch (HttpClientErrorException e) {
response = new ResponseEntity(e.getResponseBodyAsString(), e.getStatusCode());
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), e.getResponseBodyAsString()));
}
if (response == null) {
return null;
}
if (response.getStatusCode().value() == 200
|| response.getStatusCode().value() == 201
|| response.getStatusCode().value() == 404) {
return response.getBody();
} else {
String responseString = response.getBody();
if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
return responseString;
}
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), responseString));
throw new Exception(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), responseString));
}
}
public void cancelHTTP(String paragraphId) {
// TODO(zjffdu), use cancel rest api of livy
paragraphHttpMap.put(paragraphId, null);
}
public void closeSession(int sessionId) {
try {
executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId,
"DELETE", null, null);
} catch (Exception e) {
LOGGER.error(String.format("Error closing session for user with session ID: %s",
sessionId), e);
}
}
}

84
livy/src/main/java/org/apache/zeppelin/livy/LivyOutputStream.java
View file

@ -1,84 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
/**
* InterpreterOutput can be attached / detached.
*/
public class LivyOutputStream extends OutputStream {
private static Logger LOGGER = LoggerFactory.getLogger(LivyOutputStream.class);
InterpreterOutput interpreterOutput;
public LivyOutputStream() {
}
public InterpreterOutput getInterpreterOutput() {
return interpreterOutput;
}
public void setInterpreterOutput(InterpreterOutput interpreterOutput) {
this.interpreterOutput = interpreterOutput;
}
@Override
public void write(int b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte[] b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
public void write(String text) throws IOException {
LOGGER.debug("livy output:" + text);
write(text.getBytes("UTF-8"));
}
@Override
public void write(byte[] b, int offset, int len) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b, offset, len);
}
}
@Override
public void close() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.close();
}
}
@Override
public void flush() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.flush();
}
}
}

46
livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java Normal file
View file

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivyPySpark3Interpreter extends BaseLivyInterprereter {
public LivyPySpark3Interpreter(Properties property) {
super(property);
}
@Override
public String getSessionKind() {
return "pyspark3";
}
}

99
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -19,16 +19,10 @@ package org.apache.zeppelin.livy;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
/**
@ -38,10 +32,12 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
private LivySparkInterpreter sparkInterpreter;
private boolean sqlContextCreated = false;
private boolean isSpark2 = false;
private int maxResult = 1000;
public LivySparkSQLInterpreter(Properties property) {
super(property);
this.maxResult = Integer.parseInt(property.getProperty("zeppelin.livy.spark.sql.maxResult"));
}
@Override
@ -51,10 +47,56 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
@Override
public void open() {
super.open();
this.sparkInterpreter =
(LivySparkInterpreter) getInterpreterInTheSameSessionByClassName(
LivySparkInterpreter.class.getName());
this.sparkInterpreter = getSparkInterpreter();
// As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession
// to judge whether it is using spark2.
try {
InterpreterResult result = sparkInterpreter.interpret("spark", false);
if (result.code() == InterpreterResult.Code.SUCCESS &&
result.message().get(0).getData().contains("org.apache.spark.sql.SparkSession")) {
LOGGER.info("SparkSession is detected so we are using spark 2.x for session {}",
sparkInterpreter.getSessionInfo().id);
isSpark2 = true;
} else {
// spark 1.x
result = sparkInterpreter.interpret("sqlContext", false);
if (result.code() == InterpreterResult.Code.SUCCESS) {
LOGGER.info("sqlContext is detected.");
} else if (result.code() == InterpreterResult.Code.ERROR) {
// create SqlContext if it is not available, as in livy 0.2 sqlContext
// is not available.
LOGGER.info("sqlContext is not detected, try to create SQLContext by ourselves");
result = sparkInterpreter.interpret(
"val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ "import sqlContext.implicits._", false);
if (result.code() == InterpreterResult.Code.ERROR) {
throw new LivyException("Fail to create SQLContext," +
result.message().get(0).getData());
}
}
}
} catch (LivyException e) {
throw new RuntimeException("Fail to Detect SparkVersion", e);
}
}
private LivySparkInterpreter getSparkInterpreter() {
LazyOpenInterpreter lazy = null;
LivySparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
spark = (LivySparkInterpreter) p;
if (lazy != null) {
lazy.open();
}
return spark;
}
@Override
@ -64,37 +106,19 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
// create sqlContext implicitly if it is not available, as in livy 0.2 sqlContext
// is not available.
synchronized (this) {
if (!sqlContextCreated) {
InterpreterResult result = sparkInterpreter.interpret("sqlContext", context);
if (result.code() == InterpreterResult.Code.ERROR) {
result = sparkInterpreter.interpret(
"val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ "import sqlContext.implicits._", context);
if (result.code() == InterpreterResult.Code.ERROR) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Fail to create sqlContext," + result.message());
}
}
sqlContextCreated = true;
}
// use triple quote so that we don't need to do string escape.
String sqlQuery = null;
if (isSpark2) {
sqlQuery = "spark.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ")";
} else {
sqlQuery = "sqlContext.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ")";
}
// delegate the work to LivySparkInterpreter in the same session.
// TODO(zjffdu), we may create multiple session for the same user here. This can be fixed
// after we move session creation to open()
InterpreterResult res = sparkInterpreter.interpret("sqlContext.sql(\"" +
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
property.get("zeppelin.livy.spark.sql.maxResult") + ")", context);
InterpreterResult res = sparkInterpreter.interpret(sqlQuery, this.displayAppInfo);
if (res.code() == InterpreterResult.Code.SUCCESS) {
StringBuilder resMsg = new StringBuilder();
resMsg.append("%table ");
String[] rows = new String(context.out.toByteArray()).split("\n");
String[] rows = res.message().get(0).getData().split("\n");
String[] headers = rows[1].split("\\|");
for (int head = 1; head < headers.length; head++) {
resMsg.append(headers[head].trim()).append("\t");
@ -121,7 +145,6 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
} else {
return res;
}
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,

28
livy/src/main/java/org/apache/zeppelin/livy/SessionNotFoundException.java Normal file
View file

@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
/**
*
*/
public class SessionNotFoundException extends LivyException {
public SessionNotFoundException(String message) {
super(message);
}
}

29
livy/src/main/resources/interpreter-setting.json
View file

@ -13,14 +13,9 @@
},
"zeppelin.livy.create.session.retries": {
"envName": "ZEPPELIN_LIVY_CREATE_SESSION_RETRIES",
"propertyName": "zeppelin.livy.create.session.retries",
"propertyName": "zeppelin.livy.create.session.timeout",
"defaultValue": "120",
"description": "Livy Server create session retry count."
},
"livy.spark.master": {
"propertyName": "livy.spark.master",
"defaultValue": "local[*]",
"description": "Spark master uri. ex) spark://masterhost:7077"
"description": "Livy Server create session timeout (seconds)."
},
"livy.spark.driver.cores": {
"propertyName": "livy.spark.driver.cores",
@ -158,6 +153,26 @@
"editOnDblClick": false
}
},
{
"group": "livy",
"name": "pyspark3",
"className": "org.apache.zeppelin.livy.LivyPySpark3Interpreter",
"properties": {
},
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "scoped",
"isExistingProcess": false,
"setPermission": false,
"users": []
},
"editor": {
"language": "python",
"editOnDblClick": false
}
},
{
"group": "livy",
"name": "sparkr",

111
livy/src/test/java/org/apache/zeppelin/livy/LivyHelperTest.java
View file

@ -1,111 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.GsonBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.hamcrest.CoreMatchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Properties;
import static org.mockito.Mockito.doReturn;
/**
* Created for org.apache.zeppelin.livy on 22/04/16.
*/
@RunWith(MockitoJUnitRunner.class)
public class LivyHelperTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private static LivyPySparkInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext interpreterContext;
@Mock(answer = Answers.CALLS_REAL_METHODS)
private LivyHelper livyHelper;
@Before
public void prepareContext() throws Exception {
Properties properties = new Properties();
properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
livyHelper.property = properties;
livyHelper.paragraphHttpMap = new HashMap<>();
livyHelper.gson = new GsonBuilder().setPrettyPrinting().create();
livyHelper.LOGGER = LoggerFactory.getLogger(LivyHelper.class);
doReturn("{\"id\":1,\"state\":\"idle\",\"kind\":\"spark\",\"proxyUser\":\"null\",\"log\":[]}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{\"kind\": \"spark\", \"conf\": {}, \"proxyUser\": null}",
null
);
doReturn("{\"id\":1,\"state\":\"available\",\"output\":{\"status\":\"ok\"," +
"\"execution_count\":1,\"data\":{\"text/plain\":\"1\"}}}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions/1/statements",
"POST",
"{\"code\": \"print(1)\"}",
null
);
}
@Test
public void checkCreateSession() {
try {
Integer sessionId = livyHelper.createSession(interpreterContext, "spark");
collector.checkThat("check sessionId", 1, CoreMatchers.equalTo(sessionId));
} catch (Exception e) {
collector.addError(e);
}
}
@Test
public void checkInterpret() {
try {
InterpreterResult result = livyHelper.interpret("print(1)", interpreterContext, 1);
collector.checkThat("check sessionId", InterpreterResult.Code.SUCCESS,
CoreMatchers.equalTo(result.code()));
} catch (Exception e) {
collector.addError(e);
}
}
}

173
livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
View file

@ -20,6 +20,8 @@ package org.apache.zeppelin.livy;
import com.cloudera.livy.test.framework.Cluster;
import com.cloudera.livy.test.framework.Cluster$;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.*;
@ -49,13 +51,14 @@ public class LivyInterpreterIT {
LOGGER.info("Starting livy at {}", cluster.livyEndpoint());
properties = new Properties();
properties.setProperty("zeppelin.livy.url", cluster.livyEndpoint());
properties.setProperty("zeppelin.livy.create.session.retries", "120");
properties.setProperty("zeppelin.livy.create.session.timeout", "120");
properties.setProperty("zeppelin.livy.spark.sql.maxResult", "100");
}
@AfterClass
public static void tearDown() {
if (cluster != null) {
LOGGER.info("Shutting down livy at {}", cluster.livyEndpoint());
cluster.cleanUp();
}
}
@ -92,63 +95,63 @@ public class LivyInterpreterIT {
try {
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("1.5.2"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("1.5.2"));
// test RDD api
outputListener.reset();
result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("Double = 55.0"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("Double = 55.0"));
// single line comment
outputListener.reset();
String singleLineComment = "// my comment";
String singleLineComment = "println(1)// my comment";
result = sparkInterpreter.interpret(singleLineComment, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertEquals(1, result.message().size());
// multiple line comment
outputListener.reset();
String multipleLineComment = "/* multiple \n" + "line \n" + "comment */";
String multipleLineComment = "println(1)/* multiple \n" + "line \n" + "comment */";
result = sparkInterpreter.interpret(multipleLineComment, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertEquals(1, result.message().size());
// multi-line string
outputListener.reset();
String multiLineString = "val str = \"\"\"multiple\n" +
"line\"\"\"\n" +
"println(str)";
result = sparkInterpreter.interpret(multiLineString, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("multiple\nline"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("multiple\nline"));
// case class
outputListener.reset();
String caseClassCode = "case class Person(id:Int, \n" +
"name:String)\n" +
"val p=Person(1, \"name_a\")";
result = sparkInterpreter.interpret(caseClassCode, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("defined class Person"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("p: Person = Person(1,name_a)"));
// object class
outputListener.reset();
String objectClassCode = "object Person {}";
result = sparkInterpreter.interpret(objectClassCode, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("defined module Person"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("defined module Person"));
// error
result = sparkInterpreter.interpret("println(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("error: not found: value a"));
// incomplete code
result = sparkInterpreter.interpret("if(true){", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("incomplete statement"));
} finally {
sparkInterpreter.close();
}
@ -178,24 +181,46 @@ public class LivyInterpreterIT {
try {
// test DataFrame api
outputListener.reset();
sparkInterpreter.interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ "import sqlContext.implicits._", context);
InterpreterResult result = sparkInterpreter.interpret("val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n"
InterpreterResult result = sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended()
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
outputListener.reset();
result = sqlInterpreter.interpret("select * from df", context);
result = sqlInterpreter.interpret("select * from df where col_1='hello'", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
// TODO(zjffdu), \t at the end of each line is not necessary, it is a bug of LivySparkSQLInterpreter
assertEquals("_1\t_2\t\nhello\t20\t\n", result.message().get(0).getData());
// TODO(zjffdu), \t at the end of each line is not necessary,
// it is a bug of LivySparkSQLInterpreter
assertEquals("col_1\tcol_2\t\nhello\t20\t\n", result.message().get(0).getData());
// double quotes
result = sqlInterpreter.interpret("select * from df where col_1=\"hello\"", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertEquals("col_1\tcol_2\t\nhello\t20\t\n", result.message().get(0).getData());
// double quotes inside attribute value
// TODO(zjffdu). This test case would fail on spark-1.5, would uncomment it when upgrading to
// livy-0.3 and spark-1.6
// result = sqlInterpreter.interpret("select * from df where col_1=\"he\\\"llo\" ", context);
// assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
// single quotes inside attribute value
result = sqlInterpreter.interpret("select * from df where col_1=\"he'llo\"", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
// test sql with syntax error
result = sqlInterpreter.interpret("select * from df2", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("Table Not Found"));
} finally {
sparkInterpreter.close();
sqlInterpreter.close();
@ -209,10 +234,12 @@ public class LivyInterpreterIT {
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
new LivySparkInterpreter(properties));
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter(
new LivySparkSQLInterpreter(properties));
interpreterGroup.get("session_1").add(sqlInterpreter);
sqlInterpreter.setInterpreterGroup(interpreterGroup);
sqlInterpreter.open();
@ -249,26 +276,33 @@ public class LivyInterpreterIT {
try {
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("1.5.2"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("1.5.2"));
// test RDD api
outputListener.reset();
result = pysparkInterpreter.interpret("sc.range(1, 10).sum()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("45"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("45"));
// test DataFrame api
outputListener.reset();
pysparkInterpreter.interpret("from pyspark.sql import SQLContext\n"
+ "sqlContext = SQLContext(sc)", context);
result = pysparkInterpreter.interpret("df=sqlContext.createDataFrame([(\"hello\",20)])\n"
+ "df.collect()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(0, result.message().size());
assertTrue(outputListener.getOutputAppended().contains("[Row(_1=u'hello', _2=20)]"));
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]"));
// test magic api
pysparkInterpreter.interpret("t = [{\"name\":\"userA\", \"role\":\"roleA\"},"
+ "{\"name\":\"userB\", \"role\":\"roleB\"}]", context);
result = pysparkInterpreter.interpret("%table t", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("userA"));
// error
result = pysparkInterpreter.interpret("print(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
@ -287,37 +321,52 @@ public class LivyInterpreterIT {
// TODO(zjffdu), Livy's SparkRIntepreter has some issue, do it after livy-0.3 release.
}
public static class MyInterpreterOutputListener implements InterpreterOutputListener {
private StringBuilder outputAppended = new StringBuilder();
private StringBuilder outputUpdated = new StringBuilder();
@Test
public void testLivyTutorialNote() throws IOException {
if (!checkPreCondition()) {
return;
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
new LivySparkInterpreter(properties));
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter(
new LivySparkSQLInterpreter(properties));
interpreterGroup.get("session_1").add(sqlInterpreter);
sqlInterpreter.setInterpreterGroup(interpreterGroup);
sqlInterpreter.open();
try {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sql",
"title", "text", authInfo, null, null, null, null, null, output);
String p1 = IOUtils.toString(getClass().getResourceAsStream("/livy_tutorial_1.scala"));
InterpreterResult result = sparkInterpreter.interpret(p1, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
String p2 = IOUtils.toString(getClass().getResourceAsStream("/livy_tutorial_2.sql"));
result = sqlInterpreter.interpret(p2, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
} finally {
sparkInterpreter.close();
sqlInterpreter.close();
}
}
public static class MyInterpreterOutputListener implements InterpreterOutputListener {
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
LOGGER.info("onAppend:" + new String(line));
outputAppended.append(new String(line));
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
try {
LOGGER.info("onUpdate:" + new String(out.toByteArray()));
outputUpdated.append(new String(out.toByteArray()));
} catch (IOException e) {
e.printStackTrace();
}
}
public String getOutputAppended() {
return outputAppended.toString();
}
public String getOutputUpdated() {
return outputUpdated.toString();
}
public void reset() {
outputAppended = new StringBuilder();
outputUpdated = new StringBuilder();
}
@Override

24
livy/src/test/resources/livy_tutorial_1.scala Normal file
View file

@ -0,0 +1,24 @@
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)
// So you don't need create them manually
// load bank data
val bankText = sc.parallelize(
IOUtils.toString(
new URL("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv"),
Charset.forName("utf8")).split("\n"))
case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)
val bank = bankText.map(s => s.split(";")).filter(s => s(0) != "\"age\"").map(
s => Bank(s(0).toInt,
s(1).replaceAll("\"", ""),
s(2).replaceAll("\"", ""),
s(3).replaceAll("\"", ""),
s(5).replaceAll("\"", "").toInt
)
).toDF()
bank.registerTempTable("bank")

5
livy/src/test/resources/livy_tutorial_2.sql Normal file
View file

@ -0,0 +1,5 @@
select age, count(1) value
from bank
where age < 30
group by age
order by age

376
notebook/2A94M5J1Z/note.json
View file

@ -2,162 +2,165 @@
"paragraphs": [
{
"text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:32:15 PM",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
"editorHide": true,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
}
}
],
"enabled": true,
"editorSetting": {
"language": "markdown",
"editOnDblClick": true
},
"editorHide": true
"editorMode": "ace/mode/markdown",
"tableHide": false
},
"settings": {
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423836981412_-1007008116",
"id": "20150213-231621_168813393",
"result": {
"results": {
"code": "SUCCESS",
"type": "HTML",
"msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n"
"msg": [
{
"type": "HTML",
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n\u003c/div\u003e"
}
]
},
"dateCreated": "Feb 13, 2015 11:16:21 PM",
"dateStarted": "Apr 1, 2015 9:11:09 PM",
"dateFinished": "Apr 1, 2015 9:11:10 PM",
"dateStarted": "Dec 17, 2016 3:32:15 PM",
"dateFinished": "Dec 17, 2016 3:32:18 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"title": "Load data into table",
"text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")",
"dateUpdated": "Jan 14, 2016 7:58:56 PM",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:09 PM",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
},
"title": true,
"enabled": true,
"editorMode": "ace/mode/scala"
"editorMode": "ace/mode/scala",
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
],
"editorSetting": {
"language": "scala",
"editOnDblClick": false
}
},
"settings": {
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423500779206_-1502780787",
"id": "20150210-015259_1403135953",
"result": {
"results": {
"code": "SUCCESS",
"type": "TEXT",
"msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
"msg": [
{
"type": "TEXT",
"data": "\nimport org.apache.commons.io.IOUtils\n\nimport java.net.URL\n\nimport java.nio.charset.Charset\n\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[0] at parallelize at \u003cconsole\u003e:32\n\ndefined class Bank\n\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\n\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n"
}
]
},
"dateCreated": "Feb 10, 2015 1:52:59 AM",
"dateStarted": "Jul 3, 2015 1:43:40 PM",
"dateFinished": "Jul 3, 2015 1:43:45 PM",
"dateStarted": "Dec 17, 2016 3:30:09 PM",
"dateFinished": "Dec 17, 2016 3:30:58 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:13 PM",
"config": {
"colWidth": 4.0,
"graph": {
"mode": "multiBarChart",
"height": 300.0,
"optionOpen": false,
"keys": [
{
"name": "age",
"index": 0.0,
"aggr": "sum"
}
],
"values": [
{
"name": "value",
"index": 1.0,
"aggr": "sum"
}
],
"groups": [],
"scatter": {
"xAxis": {
"name": "age",
"index": 0.0,
"aggr": "sum"
},
"yAxis": {
"name": "value",
"index": 1.0,
"aggr": "sum"
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
}
],
"enabled": true,
"editorSetting": {
"language": "sql",
"editOnDblClick": false
},
"editorMode": "ace/mode/sql"
},
"settings": {
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423500782552_-1439281894",
"id": "20150210-015302_1492795503",
"result": {
"results": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
"msg": [
{
"type": "TABLE",
"data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
}
]
},
"dateCreated": "Feb 10, 2015 1:53:02 AM",
"dateStarted": "Jul 3, 2015 1:43:17 PM",
"dateFinished": "Jul 3, 2015 1:43:23 PM",
"dateStarted": "Dec 17, 2016 3:30:13 PM",
"dateFinished": "Dec 17, 2016 3:31:04 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:16 PM",
"config": {
"colWidth": 4.0,
"graph": {
"mode": "multiBarChart",
"height": 300.0,
"optionOpen": false,
"keys": [
{
"name": "age",
"index": 0.0,
"aggr": "sum"
}
],
"values": [
{
"name": "value",
"index": 1.0,
"aggr": "sum"
}
],
"groups": [],
"scatter": {
"xAxis": {
"name": "age",
"index": 0.0,
"aggr": "sum"
},
"yAxis": {
"name": "value",
"index": 1.0,
"aggr": "sum"
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
}
],
"enabled": true,
"editorSetting": {
"language": "sql",
"editOnDblClick": false
},
"editorMode": "ace/mode/sql"
},
"settings": {
"params": {
@ -171,55 +174,45 @@
}
}
},
"apps": [],
"jobName": "paragraph_1423720444030_-1424110477",
"id": "20150212-145404_867439529",
"result": {
"results": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
"msg": [
{
"type": "TABLE",
"data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
}
]
},
"dateCreated": "Feb 12, 2015 2:54:04 PM",
"dateStarted": "Jul 3, 2015 1:43:28 PM",
"dateFinished": "Jul 3, 2015 1:43:29 PM",
"dateStarted": "Dec 17, 2016 3:30:58 PM",
"dateFinished": "Dec 17, 2016 3:31:07 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:18 PM",
"config": {
"colWidth": 4.0,
"graph": {
"mode": "multiBarChart",
"height": 300.0,
"optionOpen": false,
"keys": [
{
"name": "age",
"index": 0.0,
"aggr": "sum"
}
],
"values": [
{
"name": "value",
"index": 1.0,
"aggr": "sum"
}
],
"groups": [],
"scatter": {
"xAxis": {
"name": "age",
"index": 0.0,
"aggr": "sum"
},
"yAxis": {
"name": "value",
"index": 1.0,
"aggr": "sum"
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
}
],
"enabled": true,
"editorSetting": {
"language": "sql",
"editOnDblClick": false
},
"editorMode": "ace/mode/sql"
},
"settings": {
"params": {
@ -244,80 +237,113 @@
}
}
},
"apps": [],
"jobName": "paragraph_1423836262027_-210588283",
"id": "20150213-230422_1600658137",
"result": {
"results": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
"msg": [
{
"type": "TABLE",
"data": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
}
]
},
"dateCreated": "Feb 13, 2015 11:04:22 PM",
"dateStarted": "Jul 3, 2015 1:43:33 PM",
"dateFinished": "Jul 3, 2015 1:43:34 PM",
"dateStarted": "Dec 17, 2016 3:31:05 PM",
"dateFinished": "Dec 17, 2016 3:31:09 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:24 PM",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
"editorHide": true,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
],
"enabled": true,
"editorSetting": {
"language": "markdown",
"editOnDblClick": true
},
"editorHide": true
"editorMode": "ace/mode/markdown",
"tableHide": false
},
"settings": {
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423836268492_216498320",
"id": "20150213-230428_1231780373",
"result": {
"results": {
"code": "SUCCESS",
"type": "HTML",
"msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
"msg": [
{
"type": "HTML",
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eCongratulations, it\u0026rsquo;s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0026lsquo;Notebook\u0026rsquo; menu. Good luck!\u003c/h5\u003e\n\u003c/div\u003e"
}
]
},
"dateCreated": "Feb 13, 2015 11:04:28 PM",
"dateStarted": "Apr 1, 2015 9:12:18 PM",
"dateFinished": "Apr 1, 2015 9:12:18 PM",
"dateStarted": "Dec 17, 2016 3:30:24 PM",
"dateFinished": "Dec 17, 2016 3:30:29 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:34 PM",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
"editorHide": true,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false
}
}
],
"enabled": true,
"editorSetting": {
"language": "markdown",
"editOnDblClick": true
},
"editorHide": true
"editorMode": "ace/mode/markdown",
"tableHide": false
},
"settings": {
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1427420818407_872443482",
"id": "20150326-214658_12335843",
"result": {
"results": {
"code": "SUCCESS",
"type": "HTML",
"msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
"msg": [
{
"type": "HTML",
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u0026#39;2011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n\u003c/div\u003e"
}
]
},
"dateCreated": "Mar 26, 2015 9:46:58 PM",
"dateStarted": "Jul 3, 2015 1:44:56 PM",
"dateFinished": "Jul 3, 2015 1:44:56 PM",
"dateStarted": "Dec 17, 2016 3:30:34 PM",
"dateFinished": "Dec 17, 2016 3:30:34 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
@ -327,6 +353,7 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1435955447812_-158639899",
"id": "20150703-133047_853701097",
"dateCreated": "Jul 3, 2015 1:30:47 PM",
@ -337,11 +364,28 @@
"name": "Zeppelin Tutorial/Basic Features (Spark)",
"id": "2A94M5J1Z",
"angularObjects": {
"2B6FF8NNU": [],
"2B67PH63Z": []
"2C6WUGPNH:shared_process": [],
"2C4A8RJNB:shared_process": [],
"2C4DTK2ZT:shared_process": [],
"2C6XKJWBR:shared_process": [],
"2C6AHZPMK:shared_process": [],
"2C5SU66WQ:shared_process": [],
"2C6AMJ98Q:shared_process": [],
"2C4AJZK72:shared_process": [],
"2C3STPSD7:shared_process": [],
"2C4FJN9CK:shared_process": [],
"2C3CW6JBY:shared_process": [],
"2C5UPQX6Q:shared_process": [],
"2C5873KN4:shared_process": [],
"2C5719XN4:shared_process": [],
"2C52DE5G3:shared_process": [],
"2C4G28E63:shared_process": [],
"2C6CU96BC:shared_process": [],
"2C49A6WY3:shared_process": [],
"2C3NE73HG:shared_process": []
},
"config": {
"looknfeel": "default"
},
"info": {}
}
}

1024
notebook/2BWJFTXKJ/note.json
View file

File diff suppressed because one or more lines are too long

739
notebook/2BYEZ5EVK/note.json
View file

File diff suppressed because one or more lines are too long

595
notebook/2C2AUG798/note.json
View file

File diff suppressed because one or more lines are too long

7
pig/pom.xml
View file

@ -177,6 +177,13 @@
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>always</forkMode>
</configuration>
</plugin>
</plugins>
</build>
</project>

2
pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
View file

@ -93,7 +93,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
if (schemaKnown) {
for (int i = 0; i < schema.size(); ++i) {
Schema.FieldSchema field = schema.getField(i);
resultBuilder.append(field.alias);
resultBuilder.append(field.alias != null ? field.alias : "col_" + i);
if (i != schema.size() - 1) {
resultBuilder.append("\t");
}

9
pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
View file

@ -171,6 +171,15 @@ public class PigUtils {
private static String extractFromTezPigStats(TezPigScriptStats stats) {
try {
if (stats.getReturnCode() == PigRunner.ReturnCode.UNKNOWN) {
LOGGER.warn("unknown return code, can't display the results");
return null;
}
if (stats.getPigContext() == null) {
LOGGER.warn("unknown exec type, don't display the results");
return null;
}
Field userIdField = PigStats.class.getDeclaredField("userId");
userIdField.setAccessible(true);
String userId = (String) (userIdField.get(stats));

13
pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
View file

@ -41,10 +41,10 @@ public class PigInterpreterTest {
private PigInterpreter pigInterpreter;
private InterpreterContext context;
@Before
public void setUp() {
private void setUpLocal(boolean includeJobStats) {
Properties properties = new Properties();
properties.put("zeppelin.pig.execType", "local");
properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
@ -58,6 +58,8 @@ public class PigInterpreterTest {
@Test
public void testBasics() throws IOException {
setUpLocal(false);
String content = "1\tandy\n"
+ "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
@ -101,11 +103,7 @@ public class PigInterpreterTest {
@Test
public void testIncludeJobStats() throws IOException {
Properties properties = new Properties();
properties.put("zeppelin.pig.execType", "local");
properties.put("zeppelin.pig.includeJobStats", "true");
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
setUpLocal(true);
String content = "1\tandy\n"
+ "2\tpeter\n";
@ -152,4 +150,5 @@ public class PigInterpreterTest {
assertTrue(result.message().get(0).getData().contains("Counters:"));
assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
}
}

152
pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java Normal file
View file

@ -0,0 +1,152 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.pig;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class PigInterpreterTezTest {
private PigInterpreter pigInterpreter;
private InterpreterContext context;
public void setUpTez(boolean includeJobStats) {
Properties properties = new Properties();
properties.put("zeppelin.pig.execType", "tez_local");
properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
null, null);
}
@After
public void tearDown() {
pigInterpreter.close();
}
@Test
public void testBasics() throws IOException {
setUpTez(false);
String content = "1\tandy\n"
+ "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
// syntax error
pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ "foreach a generate $0;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().contains("expecting one of"));
}
@Test
public void testIncludeJobStats() throws IOException {
setUpTez(true);
String content = "1\tandy\n"
+ "2\tpeter\n";
File tmpFile = File.createTempFile("zeppelin", "test");
FileWriter writer = new FileWriter(tmpFile);
IOUtils.write(content, writer);
writer.close();
// simple pig script using dump
String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+ "dump a;";
InterpreterResult result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains("Vertex Stats"));
assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
// describe
pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.SUCCESS, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
// syntax error (compilation error)
pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+ "describe a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
// no job is launched, so no jobStats
assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol at or near 'a'"));
// execution error
pigscript = "a = load 'invalid_path';"
+ "dump a;";
result = pigInterpreter.interpret(pigscript, context);
assertEquals(Type.TEXT, result.message().get(0).getType());
assertEquals(Code.ERROR, result.code());
assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
}
}

7
pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
View file

@ -108,6 +108,13 @@ public class PigQueryInterpreterTest {
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals("gender\tcount\nmale\t2\nfemale\t1\n", result.message().get(0).getData());
// generate alias with unknown schema
query = "b = group a by gender;\nforeach b generate group, COUNT($1);";
result = pigQueryInterpreter.interpret(query, context);
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals("group\tcol_1\nmale\t2\nfemale\t1\n", result.message().get(0).getData());
// syntax error in PigQueryInterpereter
query = "b = group a by invalid_column;\nforeach b generate group as gender, COUNT($1) as count;";
result = pigQueryInterpreter.interpret(query, context);

3
pig/src/test/resources/core-site.xml Normal file
View file

@ -0,0 +1,3 @@
<configuration>
</configuration>

2
r/R/rzeppelin/DESCRIPTION
View file

@ -9,7 +9,7 @@ URL: http://dahl.byu.edu/software/rscala/
Imports: utils,
evaluate
Suggests:
goolgeVis,
googleVis,
htmltools,
knitr,
rCharts,

12
scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java
View file

@ -57,18 +57,6 @@ public class ScaldingInterpreter extends Interpreter {
public static final List NO_COMPLETION =
Collections.unmodifiableList(new ArrayList<>());
static {
Interpreter.register(
"scalding",
"scalding",
ScaldingInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(ARGS_STRING, ARGS_STRING_DEFAULT, "Arguments for scalding REPL")
.add(MAX_OPEN_INSTANCES, MAX_OPEN_INSTANCES_DEFAULT,
"Maximum number of open interpreter instances")
.build());
}
static int numOpenInstances = 0;
private ScaldingILoop interpreter;
private ByteArrayOutputStream out;

19
scalding/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "scalding",
"name": "scalding",
"className": "org.apache.zeppelin.scalding.ScaldingInterpreter",
"properties": {
"args.string": {
"envName": null,
"defaultValue": "--local --repl",
"description": "Arguments for scalding REPL"
},
"max.open.instances": {
"envName": null,
"defaultValue": "50",
"description": "Maximum number of open interpreter instances"
}
}
}
]

7
scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala
View file

@ -35,6 +35,7 @@ private[scio] object DisplayHelpers {
private[scio] val tab = "\t"
private[scio] val newline = "\n"
private[scio] val table = "%table"
private[scio] val endTable = "%text"
private[scio] val rowLimitReachedMsg =
s"$newline<font color=red>Results are limited to " + maxResults + s" rows.</font>$newline"
private[scio] val bQSchemaIncomplete =
@ -52,6 +53,7 @@ private[scio] object DisplayHelpers {
println(sCollectionEmptyMsg)
} else {
println(s"$table value$newline${it.take(maxResults).map(printer).mkString(newline)}")
println(endTable)
notifyIfTruncated(it)
}
}
@ -64,6 +66,7 @@ private[scio] object DisplayHelpers {
println(sCollectionEmptyMsg)
} else {
println(s"$table value$newline${it.take(maxResults).map(printer).mkString(newline)}")
println(endTable)
notifyIfTruncated(it)
}
}
@ -77,6 +80,7 @@ private[scio] object DisplayHelpers {
} else {
val content = it.take(maxResults).map{ case (k, v) => s"$k$tab$v" }.mkString(newline)
println(s"$table key${tab}value$newline$content")
println(endTable)
notifyIfTruncated(it)
}
}
@ -97,6 +101,7 @@ private[scio] object DisplayHelpers {
val firstStr = first.productIterator.mkString(tab)
val content = it.take(maxResults - 1).map(_.productIterator.mkString(tab)).mkString(newline)
println(s"$table $header$newline$firstStr$newline$content")
println(endTable)
notifyIfTruncated(it)
}
}
@ -125,6 +130,7 @@ private[scio] object DisplayHelpers {
.map(r => fieldNames.map(r.get).mkString(tab))
.mkString(newline)
println(s"$table $header$newline$firstStr$newline$content")
println(endTable)
notifyIfTruncated(it)
}
}
@ -151,6 +157,7 @@ private[scio] object DisplayHelpers {
.mkString(newline)
println(s"$table $header$newline$content")
println(endTable)
notifyIfTruncated(it)
}
}

71
scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala
View file

@ -48,6 +48,7 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
// -----------------------------------------------------------------------------------------------
private val anyValHeader = s"$table value"
private val endTable = DisplayHelpers.endTable
"DisplayHelpers" should "support Integer SCollection via AnyVal" in {
import org.apache.zeppelin.scio.DisplaySCollectionImplicits.ZeppelinSCollection
@ -59,8 +60,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1",
"2",
"3")
"3",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Long SCollection via AnyVal" in {
@ -73,8 +76,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1",
"2",
"3")
"3",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Double SCollection via AnyVal" in {
@ -87,8 +92,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1.0",
"2.0",
"3.0")
"3.0",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Float SCollection via AnyVal" in {
@ -101,8 +108,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1.0",
"2.0",
"3.0")
"3.0",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Short SCollection via AnyVal" in {
@ -115,8 +124,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1",
"2",
"3")
"3",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Byte SCollection via AnyVal" in {
@ -129,8 +140,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"1",
"2",
"3")
"3",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Boolean SCollection via AnyVal" in {
@ -143,8 +156,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"true",
"false",
"true")
"true",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support Char SCollection via AnyVal" in {
@ -157,8 +172,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(anyValHeader,
"a",
"b",
"c")
"c",
endTable)
o.head should be(anyValHeader)
o.last should be(endTable)
}
it should "support SCollection of AnyVal over row limit" in {
@ -199,8 +216,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
o should contain theSameElementsAs Seq(stringHeader,
"a",
"b",
"c")
"c",
endTable)
o.head should be (stringHeader)
o.last should be (endTable)
}
it should "support empty SCollection of String" in {
@ -240,8 +259,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
}
o should contain theSameElementsAs Seq(kvHeader,
s"3${tab}4",
s"1${tab}2")
s"1${tab}2",
endTable)
o.head should be (kvHeader)
o.last should be (endTable)
}
it should "support KV (str keys) SCollection" in {
@ -253,8 +274,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
}
o should contain theSameElementsAs Seq(kvHeader,
s"foo${tab}2",
s"bar${tab}4")
s"bar${tab}4",
endTable)
o.head should be (kvHeader)
o.last should be (endTable)
}
it should "support KV (str values) SCollection" in {
@ -266,8 +289,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
}
o should contain theSameElementsAs Seq(kvHeader,
s"2${tab}foo",
s"4${tab}bar")
s"4${tab}bar",
endTable)
o.head should be (kvHeader)
o.last should be (endTable)
}
it should "support empty KV SCollection" in {
@ -305,8 +330,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(tupleHeader) ++ Seq.fill(3)(s"1${tab}2${tab}3"))
o should contain theSameElementsAs
(Seq(tupleHeader, endTable) ++ Seq.fill(3)(s"1${tab}2${tab}3"))
o.head should be(tupleHeader)
o.last should be (endTable)
}
it should "support SCollection of Tuple of 22" in {
@ -318,9 +345,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(tupleHeader) ++
o should contain theSameElementsAs (Seq(tupleHeader, endTable) ++
Seq.fill(3)((1 to 21).map(i => s"$i$tab").mkString + "22"))
o.head should be(tupleHeader)
o.last should be (endTable)
}
it should "support SCollection of Case Class of 22" in {
@ -332,9 +360,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(tupleHeader) ++
o should contain theSameElementsAs (Seq(tupleHeader, endTable) ++
Seq.fill(3)((1 to 21).map(i => s"$i$tab").mkString + "22"))
o.head should be(tupleHeader)
o.last should be (endTable)
}
it should "support SCollection of Case Class" in {
@ -344,9 +373,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(testCaseClassHeader) ++
o should contain theSameElementsAs (Seq(testCaseClassHeader, endTable) ++
Seq.fill(3)(s"1${tab}foo${tab}2.0"))
o.head should be(testCaseClassHeader)
o.last should be (endTable)
}
it should "support empty SCollection of Product" in {
@ -423,9 +453,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(avroGenericRecordHeader) ++
o should contain theSameElementsAs (Seq(avroGenericRecordHeader, endTable) ++
Seq.fill(3)(s"1${tab}1.0${tab}user1${tab}checking"))
o.head should be(avroGenericRecordHeader)
o.last should be (endTable)
}
it should "support SCollection of SpecificRecord Avro" in {
@ -436,9 +467,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay()
}
}
o should contain theSameElementsAs (Seq(avroAccountHeader) ++
o should contain theSameElementsAs (Seq(avroAccountHeader, endTable) ++
Seq.fill(3)(s"2${tab}checking${tab}user2${tab}2.0"))
o.head should be(avroAccountHeader)
o.last should be (endTable)
}
it should "support empty SCollection of SpecificRecord Avro" in {
@ -509,9 +541,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers {
in.closeAndDisplay(bQSchema)
}
}
o should contain theSameElementsAs (Seq(bQHeader) ++
o should contain theSameElementsAs (Seq(bQHeader, endTable) ++
Seq.fill(3)(s"3${tab}3.0${tab}checking${tab}user3"))
o.head should be(bQHeader)
o.last should be (endTable)
}
it should "print error on empty BQ schema" in {

42
scripts/docker/zeppelin-base/Dockerfile Normal file
View file

@ -0,0 +1,42 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM alpine:3.4
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
ENV JAVA_HOME /usr/lib/jvm/java-1.7-openjdk
ENV PATH $PATH:$JAVA_HOME/bin
RUN apk add --update bash curl openjdk7-jre wget ca-certificates python build-base make gcc g++ java-cacerts openssl && \
rm /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts && \
ln -s /etc/ssl/certs/java/cacerts /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts && \
curl --silent \
--location https://github.com/sgerrand/alpine-pkg-R/releases/download/3.3.1-r0/R-3.3.1-r0.apk --output /var/cache/apk/R-3.3.1-r0.apk && \
apk add --update --allow-untrusted /var/cache/apk/R-3.3.1-r0.apk && \
curl --silent \
--location https://github.com/sgerrand/alpine-pkg-R/releases/download/3.3.1-r0/R-dev-3.3.1-r0.apk --output /var/cache/apk/R-dev-3.3.1-r0.apk && \
apk add --update --allow-untrusted /var/cache/apk/R-dev-3.3.1-r0.apk && \
R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org')" && \
apk del curl build-base make gcc g++ && \
rm -rf /var/cache/apk/*
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64
RUN chmod +x /usr/local/bin/dumb-init
# ports for zeppelin
EXPOSE 8080 7077
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]

3
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -205,7 +205,6 @@ public class SparkInterpreter extends Interpreter {
private boolean hiveClassesArePresent() {
try {
this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState");
this.getClass().forName("org.apache.spark.sql.hive.HiveSharedState");
this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf");
return true;
} catch (ClassNotFoundException | NoClassDefFoundError e) {
@ -355,7 +354,7 @@ public class SparkInterpreter extends Interpreter {
new Class[]{ String.class, String.class},
new Object[]{ "spark.sql.catalogImplementation", "in-memory"});
sparkSession = Utils.invokeMethod(builder, "getOrCreate");
logger.info("Created Spark session with Hive support");
logger.info("Created Spark session with Hive support use in-memory catalogImplementation");
}
} else {
sparkSession = Utils.invokeMethod(builder, "getOrCreate");

36
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -42,7 +42,9 @@ public class SparkRInterpreter extends Interpreter {
private static final Logger logger = LoggerFactory.getLogger(SparkRInterpreter.class);
private static String renderOptions;
private SparkInterpreter sparkInterpreter;
private ZeppelinR zeppelinR;
private SparkContext sc;
public SparkRInterpreter(Properties property) {
super(property);
@ -60,7 +62,6 @@ public class SparkRInterpreter extends Interpreter {
// workaround to make sparkr work without SPARK_HOME
System.setProperty("spark.test.home", System.getenv("ZEPPELIN_HOME") + "/interpreter/spark");
}
synchronized (SparkRBackend.backend()) {
if (!SparkRBackend.isStarted()) {
SparkRBackend.init();
@ -70,8 +71,8 @@ public class SparkRInterpreter extends Interpreter {
int port = SparkRBackend.port();
SparkInterpreter sparkInterpreter = getSparkInterpreter();
SparkContext sc = sparkInterpreter.getSparkContext();
this.sparkInterpreter = getSparkInterpreter();
this.sc = sparkInterpreter.getSparkContext();
SparkVersion sparkVersion = new SparkVersion(sc.version());
ZeppelinRContext.setSparkContext(sc);
if (Utils.isSpark2()) {
@ -94,6 +95,10 @@ public class SparkRInterpreter extends Interpreter {
renderOptions = getProperty("zeppelin.R.render.options");
}
String getJobGroup(InterpreterContext context){
return "zeppelin-" + context.getParagraphId();
}
@Override
public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) {
@ -117,6 +122,19 @@ public class SparkRInterpreter extends Interpreter {
}
}
String jobGroup = getJobGroup(interpreterContext);
String setJobGroup = "";
// assign setJobGroup to dummy__, otherwise it would print NULL for this statement
if (Utils.isSpark2()) {
setJobGroup = "dummy__ <- setJobGroup(\"" + jobGroup +
"\", \"zeppelin sparkR job group description\", TRUE)";
} else if (getSparkInterpreter().getSparkVersion().newerThanEquals(SparkVersion.SPARK_1_5_0)) {
setJobGroup = "dummy__ <- setJobGroup(sc, \"" + jobGroup +
"\", \"zeppelin sparkR job group description\", TRUE)";
}
logger.debug("set JobGroup:" + setJobGroup);
lines = setJobGroup + "\n" + lines;
try {
// render output with knitr
if (useKnitr()) {
@ -155,7 +173,11 @@ public class SparkRInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {}
public void cancel(InterpreterContext context) {
if (this.sc != null) {
sc.cancelJobGroup(getJobGroup(context));
}
}
@Override
public FormType getFormType() {
@ -164,7 +186,11 @@ public class SparkRInterpreter extends Interpreter {
@Override
public int getProgress(InterpreterContext context) {
return 0;
if (sparkInterpreter != null) {
return sparkInterpreter.getProgress(context);
} else {
return 0;
}
}
@Override

4
spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
View file

@ -34,10 +34,10 @@ public class SparkVersion {
public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0");
public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0");
public static final SparkVersion SPARK_2_1_0 = SparkVersion.fromVersionString("2.1.0");
public static final SparkVersion SPARK_2_2_0 = SparkVersion.fromVersionString("2.2.0");
public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_1_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_2_0;
private int version;
private String versionString;

3
spark/src/main/resources/R/zeppelin_sparkr.R
View file

@ -31,7 +31,7 @@ print(paste("LibPath ", libPath))
library(SparkR)
SparkR:::connectBackend("localhost", port)
SparkR:::connectBackend("localhost", port, 6000)
# scStartTime is needed by R/pkg/R/sparkR.R
assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv)
@ -45,6 +45,7 @@ assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
if (version >= 20000) {
assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv)
assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
assign(".sparkRjsc", get(".sc", envir = SparkR:::.sparkREnv), envir=SparkR:::.sparkREnv)
}
assign(".sqlc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSqlContext"), envir = SparkR:::.sparkREnv)
assign("sqlContext", get(".sqlc", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)

12
spark/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala
View file

@ -18,15 +18,15 @@
package org.apache.zeppelin.spark
import org.apache.zeppelin.interpreter.InterpreterResult.Code
import org.apache.zeppelin.interpreter.InterpreterResult.Code.{SUCCESS, ERROR}
import org.apache.zeppelin.interpreter.InterpreterResult.Code.{SUCCESS}
import org.apache.zeppelin.interpreter.InterpreterResult.Type
import org.apache.zeppelin.interpreter.InterpreterResult.Type.{TEXT, HTML, TABLE, IMG}
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.jsoup.nodes.Document
import org.jsoup.nodes.Document.OutputSettings
import org.jsoup.safety.Whitelist
import scala.collection.JavaConversions._
import scala.util.matching.Regex
case class RDisplay(content: String, `type`: Type, code: Code)
@ -64,11 +64,13 @@ object ZeppelinRDisplay {
}
return htmlDisplay(body, imageWidth)
}
private def textDisplay(body: Element): RDisplay = {
RDisplay(body.getElementsByTag("p").first().html(), TEXT, SUCCESS)
// remove HTML tag while preserving whitespaces and newlines
val text = Jsoup.clean(body.html(), "",
Whitelist.none(), new OutputSettings().prettyPrint(false))
RDisplay(text, TEXT, SUCCESS)
}
private def tableDisplay(body: Element): RDisplay = {

38
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java
View file

@ -16,20 +16,18 @@
*/
package org.apache.zeppelin.spark;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.*;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
@ -40,10 +38,13 @@ import static org.junit.Assert.*;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class PySparkInterpreterMatplotlibTest {
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
public static SparkInterpreter sparkInterpreter;
public static PySparkInterpreter pyspark;
public static InterpreterGroup intpGroup;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class);
private InterpreterContext context;
@ -79,7 +80,7 @@ public class PySparkInterpreterMatplotlibTest {
}
}
public static Properties getPySparkTestProperties() {
private Properties getPySparkTestProperties() throws IOException {
Properties p = new Properties();
p.setProperty("master", "local[*]");
p.setProperty("spark.app.name", "Zeppelin Test");
@ -87,6 +88,7 @@ public class PySparkInterpreterMatplotlibTest {
p.setProperty("zeppelin.spark.maxResult", "1000");
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.pyspark.python", "python");
p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
return p;
}
@ -106,10 +108,6 @@ public class PySparkInterpreterMatplotlibTest {
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
@ -137,24 +135,6 @@ public class PySparkInterpreterMatplotlibTest {
new InterpreterOutput(null));
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
private void delete(File file) {
if (file.isFile()) file.delete();
else if (file.isDirectory()) {
File[] files = file.listFiles();
if (files != null && files.length > 0) {
for (File f : files) {
delete(f);
}
}
file.delete();
}
}
@Test
public void dependenciesAreInstalled() {
// matplotlib

36
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java
View file

@ -16,20 +16,22 @@
*/
package org.apache.zeppelin.spark;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -39,14 +41,17 @@ import static org.junit.Assert.*;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class PySparkInterpreterTest {
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
public static SparkInterpreter sparkInterpreter;
public static PySparkInterpreter pySparkInterpreter;
public static InterpreterGroup intpGroup;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class);
private InterpreterContext context;
public static Properties getPySparkTestProperties() {
private Properties getPySparkTestProperties() throws IOException {
Properties p = new Properties();
p.setProperty("master", "local[*]");
p.setProperty("spark.app.name", "Zeppelin Test");
@ -54,6 +59,7 @@ public class PySparkInterpreterTest {
p.setProperty("zeppelin.spark.maxResult", "1000");
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.pyspark.python", "python");
p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
return p;
}
@ -73,10 +79,6 @@ public class PySparkInterpreterTest {
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
@ -104,24 +106,6 @@ public class PySparkInterpreterTest {
new InterpreterOutput(null));
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
private void delete(File file) {
if (file.isFile()) file.delete();
else if (file.isDirectory()) {
File[] files = file.listFiles();
if (files != null && files.length > 0) {
for (File f : files) {
delete(f);
}
}
file.delete();
}
}
@Test
public void testBasicIntp() {
if (getSparkVersionNumber() > 11) {

54
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -19,7 +19,7 @@ package org.apache.zeppelin.spark;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -35,20 +35,24 @@ import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class SparkInterpreterTest {
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
public static SparkInterpreter repl;
public static InterpreterGroup intpGroup;
private InterpreterContext context;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterTest.class);
/**
@ -65,28 +69,24 @@ public class SparkInterpreterTest {
return version;
}
public static Properties getSparkTestProperties() {
public static Properties getSparkTestProperties(TemporaryFolder tmpDir) throws IOException {
Properties p = new Properties();
p.setProperty("master", "local[*]");
p.setProperty("spark.app.name", "Zeppelin Test");
p.setProperty("zeppelin.spark.useHiveContext", "true");
p.setProperty("zeppelin.spark.maxResult", "1000");
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
return p;
}
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
if (repl == null) {
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
repl = new SparkInterpreter(getSparkTestProperties());
repl = new SparkInterpreter(getSparkTestProperties(tmpDir));
repl.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl);
repl.open();
@ -102,24 +102,6 @@ public class SparkInterpreterTest {
new InterpreterOutput(null));
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
private void delete(File file) {
if (file.isFile()) file.delete();
else if (file.isDirectory()) {
File[] files = file.listFiles();
if (files != null && files.length > 0) {
for (File f : files) {
delete(f);
}
}
file.delete();
}
}
@Test
public void testBasicIntp() {
assertEquals(InterpreterResult.Code.SUCCESS,
@ -194,7 +176,7 @@ public class SparkInterpreterTest {
}
@Test
public void testSparkSql(){
public void testSparkSql() throws IOException {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context);
assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code());
@ -202,7 +184,7 @@ public class SparkInterpreterTest {
if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default.
// create new interpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties(tmpDir));
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
@ -236,9 +218,9 @@ public class SparkInterpreterTest {
}
@Test
public void shareSingleSparkContext() throws InterruptedException {
public void shareSingleSparkContext() throws InterruptedException, IOException {
// create another SparkInterpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties(tmpDir));
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
@ -252,10 +234,10 @@ public class SparkInterpreterTest {
}
@Test
public void testEnableImplicitImport() {
public void testEnableImplicitImport() throws IOException {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "true", and initialize new Spark repl
Properties p = getSparkTestProperties();
Properties p = getSparkTestProperties(tmpDir);
p.setProperty("zeppelin.spark.importImplicit", "true");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
@ -269,11 +251,11 @@ public class SparkInterpreterTest {
}
@Test
public void testDisableImplicitImport() {
public void testDisableImplicitImport() throws IOException {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "false", and initialize new Spark repl
// this test should return error status when creating DataFrame from sequence
Properties p = getSparkTestProperties();
Properties p = getSparkTestProperties(tmpDir);
p.setProperty("zeppelin.spark.importImplicit", "false");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);

21
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -17,8 +17,6 @@
package org.apache.zeppelin.spark;
import static org.junit.Assert.*;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
@ -29,25 +27,28 @@ import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.rules.TemporaryFolder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class SparkSqlInterpreterTest {
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
private SparkSqlInterpreter sql;
private SparkInterpreter repl;
private InterpreterContext context;
private InterpreterGroup intpGroup;
Logger LOGGER = LoggerFactory.getLogger(SparkSqlInterpreterTest.class);
@Before
public void setUp() throws Exception {
Properties p = new Properties();
p.putAll(SparkInterpreterTest.getSparkTestProperties());
p.putAll(SparkInterpreterTest.getSparkTestProperties(tmpDir));
p.setProperty("zeppelin.spark.maxResult", "1000");
p.setProperty("zeppelin.spark.concurrentSQL", "false");
p.setProperty("zeppelin.spark.sql.stacktrace", "false");
@ -82,10 +83,6 @@ public class SparkSqlInterpreterTest {
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
}
@After
public void tearDown() throws Exception {
}
boolean isDataFrameSupported() {
return SparkInterpreterTest.getSparkVersionNumber() >= 13;
}

6
testing/install_external_dependencies.sh
View file

@ -27,7 +27,9 @@ if [[ ${PROFILE/"-Pr "} != $PROFILE ]] || [[ ${PROFILE/"-Psparkr "} != $PROFILE
source ~/.environ
if [[ ! -d "$HOME/R/knitr" ]] ; then
mkdir -p ~/R
R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
R -e "install.packages('evaluate', repos = 'http://cran.us.r-project.org', lib='~/R')" > /dev/null 2>&1
R -e "install.packages('base64enc', repos = 'http://cran.us.r-project.org', lib='~/R')" > /dev/null 2>&1
R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')" > /dev/null 2>&1
fi
fi
@ -42,5 +44,5 @@ if [[ -n "$PYTHON" ]] ; then
conda update -q conda
conda info -a
conda config --add channels conda-forge
conda install -q matplotlib pandasql
conda install -q matplotlib=1.5.3 pandasql
fi

28
testing/setupLivy.sh Executable file
View file

@ -0,0 +1,28 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -xe
if [[ -n $LIVY_VER ]]; then
./testing/downloadLivy.sh
export LIVY_HOME=`pwd`/livy-server-$LIVY_VER
export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER
fi
set +xe

2
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -17,7 +17,6 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.4 - http://commons.apache.org/proper/commons-lang/)
(Apache 2.0) Apache Commons Math 3 (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
(Apache 2.0) Apache Commons Net (commons-net:commons-net:2.2 - http://commons.apache.org/proper/commons-net/)
(Apache 2.0) Apache log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/)
(Apache 2.0) Apache Commons Pool2 (commons-exec:commons-pool2:2.3 - https://commons.apache.org/proper/commons-pool/)
(Apache 2.0) Apache Commons FileUpload (commons-fileupload:commons-fileupload:1.3.1 - http://commons.apache.org/fileupload/)
(Apache 2.0) Apache Commons IO (commons-io:commons-io:2.4 - http://commons.apache.org/io/)
@ -266,6 +265,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) Java String Similarity 0.12 (info.debatty:java-string-similarity:0.12 - https://github.com/tdebatty/java-string-similarity)
(The MIT License) Java LSH 0.10 (info.debatty:java-lsh:0.10 - https://github.com/tdebatty/java-LSH)
(The MIT License) JSoup 1.6.1 (org.jsoup:jsoup:1.6.1 - https://github.com/jhy/jsoup/)
(The MIT License) ngclipboard v1.1.1 (https://github.com/sachinchoolur/ngclipboard) - https://github.com/sachinchoolur/ngclipboard/blob/1.1.1/LICENSE)
========================================================================
BSD-style licenses

37
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java
View file

@ -17,14 +17,15 @@
package org.apache.zeppelin.dep;
import java.io.File;
import org.apache.commons.lang3.Validate;
import org.apache.maven.repository.internal.MavenRepositorySystemSession;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.repository.LocalRepository;
import org.sonatype.aether.repository.RemoteRepository;
import java.nio.file.Paths;
/**
* Manage mvn repository.
*/
@ -35,21 +36,11 @@ public class Booter {
public static RepositorySystemSession newRepositorySystemSession(
RepositorySystem system, String localRepoPath) {
Validate.notNull(localRepoPath, "localRepoPath should have a value");
MavenRepositorySystemSession session = new MavenRepositorySystemSession();
// find homedir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
String path = home + "/" + localRepoPath;
LocalRepository localRepo =
new LocalRepository(new File(path).getAbsolutePath());
LocalRepository localRepo = new LocalRepository(resolveLocalRepoPath(localRepoPath));
session.setLocalRepositoryManager(system.newLocalRepositoryManager(localRepo));
// session.setTransferListener(new ConsoleTransferListener());
@ -61,10 +52,24 @@ public class Booter {
return session;
}
static String resolveLocalRepoPath(String localRepoPath) {
// todo decouple home folder resolution
// find homedir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
return Paths.get(home).resolve(localRepoPath).toAbsolutePath().toString();
}
public static RemoteRepository newCentralRepository() {
return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/");
}
public static RemoteRepository newLocalRepository() {
return new RemoteRepository("local",
"default", "file://" + System.getProperty("user.home") + "/.m2/repository");

20
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java
View file

@ -389,7 +389,7 @@ public class Input implements Serializable {
List<String> splits = new ArrayList<>();
String curString = "";
StringBuilder curString = new StringBuilder();
boolean escape = false; // true when escape char is found
int lastEscapeOffset = -1;
@ -408,16 +408,16 @@ public class Input implements Serializable {
// escaped char comes
if (escape == true) {
if (escapeSeq.indexOf(c) < 0) {
curString += escapeChar;
curString.append(escapeChar);
}
curString += c;
curString.append(c);
escape = false;
lastEscapeOffset = curString.length();
continue;
}
if (blockStack.size() > 0) { // inside of block
curString += c;
curString.append(c);
// check multichar block
boolean multicharBlockDetected = false;
for (int b = 0; b < blockStart.length; b++) {
@ -453,11 +453,11 @@ public class Input implements Serializable {
if (isNestedBlock(blockEnd[blockStack.get(0)]) == false) {
for (String splitter : splitters) {
if (splitter.compareTo(getBlockStr(blockEnd[blockStack.get(0)])) == 0) {
splits.add(curString);
splits.add(curString.toString());
if (includeSplitter == true) {
splits.add(splitter);
}
curString = "";
curString.setLength(0);
lastEscapeOffset = -1;
break;
@ -475,11 +475,11 @@ public class Input implements Serializable {
// forward check for splitter
int curentLenght = i + splitter.length();
if (splitter.compareTo(str.substring(i, Math.min(curentLenght, str.length()))) == 0) {
splits.add(curString);
splits.add(curString.toString());
if (includeSplitter == true) {
splits.add(splitter);
}
curString = "";
curString.setLength(0);
lastEscapeOffset = -1;
i += splitter.length() - 1;
splitted = true;
@ -491,7 +491,7 @@ public class Input implements Serializable {
}
// add char to current string
curString += c;
curString.append(c);
// check if block is started
for (int b = 0; b < blockStart.length; b++) {
@ -505,7 +505,7 @@ public class Input implements Serializable {
}
}
if (curString.length() > 0) {
splits.add(curString.trim());
splits.add(curString.toString().trim());
}
return splits.toArray(new String[] {});

45
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.google.gson.annotations.SerializedName;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.annotation.Experimental;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@ -44,7 +43,6 @@ import org.slf4j.LoggerFactory;
* open(), close(), interpret() is three the most important method you need to implement.
* cancel(), getProgress(), completion() is good to have
* getFormType(), getScheduler() determine Zeppelin's behavior
*
*/
public abstract class Interpreter {
@ -66,16 +64,12 @@ public abstract class Interpreter {
* Run code and return result, in synchronous way.
*
* @param st statements to run
* @param context
* @return
*/
@ZeppelinApi
public abstract InterpreterResult interpret(String st, InterpreterContext context);
/**
* Optionally implement the canceling routine to abort interpret() method
*
* @param context
*/
@ZeppelinApi
public abstract void cancel(InterpreterContext context);
@ -85,7 +79,7 @@ public abstract class Interpreter {
* see http://zeppelin.apache.org/docs/dynamicform.html
*
* @return FormType.SIMPLE enables simple pattern replacement (eg. Hello ${name=world}),
* FormType.NATIVE handles form in API
* FormType.NATIVE handles form in API
*/
@ZeppelinApi
public abstract FormType getFormType();
@ -93,7 +87,6 @@ public abstract class Interpreter {
/**
* get interpret() method running process in percentage.
*
* @param context
* @return number between 0-100
*/
@ZeppelinApi
@ -121,26 +114,17 @@ public abstract class Interpreter {
* SchedulerFactory.singleton().createOrGetFIFOScheduler()
* SchedulerFactory.singleton().createOrGetParallelScheduler()
*
*
* @return return scheduler instance.
* This method can be called multiple times and have to return the same instance.
* Can not return null.
* @return return scheduler instance. This method can be called multiple times and have to return
* the same instance. Can not return null.
*/
@ZeppelinApi
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
}
/**
* Called when interpreter is no longer used.
*/
@ZeppelinApi
public void destroy() {
}
public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
private InterpreterGroup interpreterGroup;
private URL [] classloaderUrls;
private URL[] classloaderUrls;
protected Properties property;
private String userName;
@ -215,6 +199,7 @@ public abstract class Interpreter {
/**
* General function to register hook event
*
* @param noteId - Note to bind hook to
* @param event The type of event to hook to (pre_exec, post_exec)
* @param cmd The code to be executed by the interpreter on given event
@ -228,6 +213,7 @@ public abstract class Interpreter {
/**
* registerHook() wrapper for global scope
*
* @param event The type of event to hook to (pre_exec, post_exec)
* @param cmd The code to be executed by the interpreter on given event
*/
@ -238,6 +224,7 @@ public abstract class Interpreter {
/**
* Get the hook code
*
* @param noteId - Note to bind hook to
* @param event The type of event to hook to (pre_exec, post_exec)
*/
@ -250,6 +237,7 @@ public abstract class Interpreter {
/**
* getHook() wrapper for global scope
*
* @param event The type of event to hook to (pre_exec, post_exec)
*/
@Experimental
@ -259,6 +247,7 @@ public abstract class Interpreter {
/**
* Unbind code from given hook event
*
* @param noteId - Note to bind hook to
* @param event The type of event to hook to (pre_exec, post_exec)
*/
@ -271,13 +260,14 @@ public abstract class Interpreter {
/**
* unregisterHook() wrapper for global scope
*
* @param event The type of event to hook to (pre_exec, post_exec)
*/
@Experimental
public void unregisterHook(String event) {
unregisterHook(null, event);
}
@ZeppelinApi
public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
synchronized (interpreterGroup) {
@ -318,17 +308,16 @@ public abstract class Interpreter {
* Represent registered interpreter class
*/
public static class RegisteredInterpreter {
//@SerializedName("interpreterGroup")
private String group;
//@SerializedName("interpreterName")
private String name;
//@SerializedName("interpreterClassName")
private String className;
private boolean defaultInterpreter;
private Map<String, InterpreterProperty> properties;
private Map<String, Object> editor;
private String path;
private InterpreterOption option;
private InterpreterRunner runner;
public RegisteredInterpreter(String name, String group, String className,
Map<String, InterpreterProperty> properties) {
@ -389,6 +378,10 @@ public abstract class Interpreter {
public InterpreterOption getOption() {
return option;
}
public InterpreterRunner getRunner() {
return runner;
}
}
/**
@ -424,8 +417,8 @@ public abstract class Interpreter {
public static void register(String name, String group, String className,
boolean defaultInterpreter, Map<String, InterpreterProperty> properties) {
logger.warn("Static initialization is deprecated for interpreter {}, You should change it " +
"to use interpreter-setting.json in your jar or " +
"interpreter/{interpreter}/interpreter-setting.json", name);
"to use interpreter-setting.json in your jar or " +
"interpreter/{interpreter}/interpreter-setting.json", name);
register(new RegisteredInterpreter(name, group, className, defaultInterpreter, properties));
}

89
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -138,8 +138,6 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.remoteInterpreterProcess = remoteInterpreterProcess;
}
/**
* Close all interpreter instances in this group
*/
@ -150,6 +148,15 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
intpToClose.addAll(intpGroupForSession);
}
close(intpToClose);
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
}
/**
@ -160,6 +167,14 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
List<Interpreter> intpForSession = this.get(sessionId);
close(intpForSession);
if (remoteInterpreterProcess != null) {
remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
}
private void close(Collection<Interpreter> intpToClose) {
@ -193,76 +208,6 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
}
}
/**
* Destroy all interpreter instances in this group for the session
* @param sessionId
*/
public void destroy(String sessionId) {
LOGGER.info("Destroy interpreter group " + getId() + " for session " + sessionId);
List<Interpreter> intpForSession = this.get(sessionId);
destroy(intpForSession);
if (remoteInterpreterProcess != null) {
remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
}
/**
* Destroy all interpreter instances in this group
*/
public void destroy() {
LOGGER.info("Destroy interpreter group " + getId());
List<Interpreter> intpToDestroy = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToDestroy.addAll(intpGroupForSession);
}
destroy(intpToDestroy);
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
}
private void destroy(Collection<Interpreter> intpToDestroy) {
if (intpToDestroy == null) {
return;
}
List<Thread> destroyThreads = new LinkedList<>();
for (final Interpreter intp : intpToDestroy) {
Thread t = new Thread() {
public void run() {
intp.destroy();
}
};
t.start();
destroyThreads.add(t);
}
for (Thread t : destroyThreads) {
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreter", e);
}
}
}
public void setResourcePool(ResourcePool resourcePool) {
this.resourcePool = resourcePool;
}

18
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java Normal file
View file

@ -0,0 +1,18 @@
package org.apache.zeppelin.interpreter;
import com.google.gson.annotations.SerializedName;
/**
* Interpreter runner path
*/
public class InterpreterRunner {
@SerializedName("linux")
private String linuxPath;
@SerializedName("win")
private String winPath;
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}
}

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java
View file

@ -31,7 +31,7 @@ public class LazyOpenInterpreter
extends Interpreter
implements WrappedInterpreter {
private Interpreter intp;
boolean opened = false;
volatile boolean opened = false;
public LazyOpenInterpreter(Interpreter intp) {
super(new Properties());
@ -59,7 +59,7 @@ public class LazyOpenInterpreter
}
@Override
public void open() {
public synchronized void open() {
if (opened == true) {
return;
}
@ -107,8 +107,11 @@ public class LazyOpenInterpreter
@Override
public int getProgress(InterpreterContext context) {
open();
return intp.getProgress(context);
if (opened) {
return intp.getProgress(context);
} else {
return 0;
}
}
@Override

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
View file

@ -70,9 +70,9 @@ public class ClientFactory extends BasePooledObjectFactory<Client>{
@Override
public void destroyObject(PooledObject<Client> p) {
synchronized (clientSocketMap) {
if (clientSocketMap.containsKey(p)) {
clientSocketMap.get(p).close();
clientSocketMap.remove(p);
if (clientSocketMap.containsKey(p.getObject())) {
clientSocketMap.get(p.getObject()).close();
clientSocketMap.remove(p.getObject());
}
}
}

94
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -44,17 +44,18 @@ import com.google.gson.reflect.TypeToken;
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener applicationEventListener;
Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
Gson gson = new Gson();
private Gson gson = new Gson();
private String interpreterRunner;
private String interpreterPath;
private String localRepoPath;
private String className;
private String sessionKey;
FormType formType;
boolean initialized;
private FormType formType;
private boolean initialized;
private Map<String, String> env;
private int connectTimeout;
private int maxPoolSize;
@ -66,18 +67,10 @@ public class RemoteInterpreter extends Interpreter {
/**
* Remote interpreter and manage interpreter process
*/
public RemoteInterpreter(Properties property,
String sessionKey,
String className,
String interpreterRunner,
String interpreterPath,
String localRepoPath,
int connectTimeout,
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener,
String userName,
Boolean isUserImpersonate) {
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
@ -98,24 +91,17 @@ public class RemoteInterpreter extends Interpreter {
/**
* Connect to existing process
*/
public RemoteInterpreter(
Properties property,
String sessionKey,
String className,
String host,
int port,
int connectTimeout,
int maxPoolSize,
public RemoteInterpreter(Properties property, String sessionKey, String className, String host,
int port, String localRepoPath, int connectTimeout, int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener,
String userName,
Boolean isUserImpersonate) {
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
initialized = false;
this.host = host;
this.port = port;
this.localRepoPath = localRepoPath;
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
@ -126,19 +112,11 @@ public class RemoteInterpreter extends Interpreter {
// VisibleForTesting
public RemoteInterpreter(
Properties property,
String sessionKey,
String className,
String interpreterRunner,
String interpreterPath,
String localRepoPath,
Map<String, String> env,
int connectTimeout,
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath,
Map<String, String> env, int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener,
String userName,
Boolean isUserImpersonate) {
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
super(property);
this.className = className;
this.sessionKey = sessionKey;
@ -240,7 +218,7 @@ public class RemoteInterpreter extends Interpreter {
property.put("zeppelin.interpreter.localRepo", localRepoPath);
}
client.createInterpreter(groupId, sessionKey,
getClassName(), (Map) property, userName);
getClassName(), (Map) property, userName);
// Push angular object loaded from JSON file to remote interpreter
if (!interpreterGroup.isAngularRegistryPushed()) {
pushAngularObjectRegistryToRemote(client);
@ -259,7 +237,6 @@ public class RemoteInterpreter extends Interpreter {
}
@Override
public void open() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
@ -347,7 +324,6 @@ public class RemoteInterpreter extends Interpreter {
context.getConfig().clear();
context.getConfig().putAll(remoteConfig);
if (form == FormType.NATIVE) {
GUI remoteGui = gson.fromJson(remoteResult.getGui(), GUI.class);
currentGUI.clear();
@ -394,7 +370,6 @@ public class RemoteInterpreter extends Interpreter {
}
}
@Override
public FormType getFormType() {
init();
@ -480,9 +455,7 @@ public class RemoteInterpreter extends Interpreter {
} else {
return SchedulerFactory.singleton().createOrGetRemoteScheduler(
RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(),
sessionKey,
interpreterProcess,
maxConcurrency);
sessionKey, interpreterProcess, maxConcurrency);
}
}
@ -491,16 +464,9 @@ public class RemoteInterpreter extends Interpreter {
}
private RemoteInterpreterContext convert(InterpreterContext ic) {
return new RemoteInterpreterContext(
ic.getNoteId(),
ic.getParagraphId(),
ic.getReplName(),
ic.getParagraphTitle(),
ic.getParagraphText(),
gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()),
gson.toJson(ic.getGui()),
gson.toJson(ic.getRunners()));
return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners()));
}
private InterpreterResult convert(RemoteInterpreterResult result) {
@ -518,22 +484,21 @@ public class RemoteInterpreter extends Interpreter {
* Push local angular object registry to
* remote interpreter. This method should be
* call ONLY inside the init() method
* @param client
* @throws TException
*/
void pushAngularObjectRegistryToRemote(Client client) throws TException {
final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
.getAngularObjectRegistry();
.getAngularObjectRegistry();
if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
.getRegistry();
.getRegistry();
logger.info("Push local angular object registry from ZeppelinServer to" +
" remote interpreter group {}", this.getInterpreterGroup().getId());
" remote interpreter group {}", this.getInterpreterGroup().getId());
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {}.getType();
Map<String, AngularObject>>>() {
}.getType();
Gson gson = new Gson();
client.angularRegistryPush(gson.toJson(registry, registryType));
@ -554,4 +519,9 @@ public class RemoteInterpreter extends Interpreter {
}
this.env.putAll(env);
}
//Only for test
public String getInterpreterRunner() {
return interpreterRunner;
}
}

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -106,7 +106,6 @@ public class RemoteInterpreterServer
eventClient.waitForEventQueueBecomesEmpty();
if (interpreterGroup != null) {
interpreterGroup.close();
interpreterGroup.destroy();
}
server.stop();
@ -403,6 +402,7 @@ public class RemoteInterpreterServer
private String script;
private InterpreterContext context;
private Map<String, Object> infos;
private Object results;
public InterpretJob(
String jobId,
@ -418,6 +418,11 @@ public class RemoteInterpreterServer
this.context = context;
}
@Override
public Object getReturn() {
return results;
}
@Override
public int progress() {
return 0;
@ -515,6 +520,11 @@ public class RemoteInterpreterServer
protected boolean jobAbort() {
return false;
}
@Override
public void setResult(Object results) {
this.results = results;
}
}

54
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -28,15 +28,14 @@ import org.slf4j.LoggerFactory;
/**
* Skeletal implementation of the Job concept.
* - designed for inheritance
* - should be run on a separate thread
* - maintains internal state: it's status
* - supports listeners who are updated on status change
*
* Job class is serialized/deserialized and used server<->client communication
* and saving/loading jobs from disk.
* Changing/adding/deleting non transitive field name need consideration of that.
* - designed for inheritance
* - should be run on a separate thread
* - maintains internal state: it's status
* - supports listeners who are updated on status change
*
* Job class is serialized/deserialized and used server<->client communication
* and saving/loading jobs from disk.
* Changing/adding/deleting non transitive field name need consideration of that.
*/
public abstract class Job {
/**
@ -48,15 +47,10 @@ public abstract class Job {
* FINISHED - Job finished run. with success
* ERROR - Job finished run. with error
* ABORT - Job finished by abort
*
*/
public static enum Status {
READY,
PENDING,
RUNNING,
FINISHED,
ERROR,
ABORT;
READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public boolean isReady() {
return this == READY;
}
@ -70,16 +64,10 @@ public abstract class Job {
}
}
private String jobName;
String id;
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
Object results;
// For backward compatibility of note.json format after ZEPPELIN-212
Object result;
Date dateCreated;
Date dateStarted;
Date dateFinished;
@ -125,6 +113,10 @@ public abstract class Job {
setStatus(Status.READY);
}
public void setId(String id) {
this.id = id;
}
public String getId() {
return id;
}
@ -180,7 +172,7 @@ public abstract class Job {
progressUpdator = new JobProgressPoller(this, progressUpdateIntervalMs);
progressUpdator.start();
dateStarted = new Date();
results = jobRun();
setResult(jobRun());
this.exception = null;
errorMessage = null;
dateFinished = new Date();
@ -189,14 +181,14 @@ public abstract class Job {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
results = e.getMessage();
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} catch (Throwable e) {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
results = e.getMessage();
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} finally {
@ -222,13 +214,7 @@ public abstract class Job {
errorMessage = getStack(t);
}
public Object getPreviousResultFormat() {
return result;
}
public Object getReturn() {
return results;
}
public abstract Object getReturn();
public String getJobName() {
return jobName;
@ -266,7 +252,5 @@ public abstract class Job {
return dateFinished;
}
public void setResult(Object results) {
this.results = results;
}
public abstract void setResult(Object results);
}

49
zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/BooterTest.java Normal file
View file

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import org.junit.Test;
import java.nio.file.Paths;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
public class BooterTest {
@Test
public void should_return_absolute_path() {
String resolvedPath = Booter.resolveLocalRepoPath("path");
assertTrue(Paths.get(resolvedPath).isAbsolute());
}
@Test
public void should_not_change_absolute_path() {
String absolutePath
= Paths.get("first", "second").toAbsolutePath().toString();
String resolvedPath = Booter.resolveLocalRepoPath(absolutePath);
assertThat(resolvedPath, equalTo(absolutePath));
}
@Test(expected = NullPointerException.class)
public void should_throw_exception_for_null() {
Booter.resolveLocalRepoPath(null);
}
}

1
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -103,7 +103,6 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
public void tearDown() throws Exception {
intp.close();
intpGroup.close();
intpGroup.destroy();
}
@Test

1
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -58,7 +58,6 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@After
public void tearDown() throws Exception {
intpGroup.close();
intpGroup.destroy();
}
private RemoteInterpreter createMockInterpreter() {

57
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

@ -73,7 +73,6 @@ public class RemoteInterpreterTest {
@After
public void tearDown() throws Exception {
intpGroup.close();
intpGroup.destroy();
}
private RemoteInterpreter createMockInterpreterA(Properties p) {
@ -299,6 +298,17 @@ public class RemoteInterpreterTest {
long start = System.currentTimeMillis();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
@ -337,6 +347,18 @@ public class RemoteInterpreterTest {
Job jobB = new Job("jobB", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
@ -404,6 +426,17 @@ public class RemoteInterpreterTest {
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
@ -484,6 +517,17 @@ public class RemoteInterpreterTest {
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
@ -587,6 +631,17 @@ public class RemoteInterpreterTest {
intpA.open();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -134,11 +134,9 @@ public class DistributedResourcePoolTest {
eventPoller1.shutdown();
intp1.close();
intpGroup1.close();
intpGroup1.destroy();
eventPoller2.shutdown();
intp2.close();
intpGroup2.close();
intpGroup2.destroy();
}
@Test

Some files were not shown because too many files have changed in this diff Show more