Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

merge master

Browse source
This commit is contained in:
karuppayya 2016-01-21 20:55:58 +05:30
parent 4fd2113fda
commit 825f696f48
162 changed files with 5104 additions and 1966 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
.travis.yml
View file

@ -14,61 +14,46 @@
# limitations under the License.
language: java
jdk:
- oraclejdk7
matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr"
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.2.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
install:
- mvn package -DskipTests -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- mvn $BUILD_FLAG $PROFILE -B
before_script:
-
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
# spark 1.6
- mvn package -Pbuild-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- ./testing/startSparkCluster.sh 1.6.0 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.6.0-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn verify -Pusing-packaged-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- ./testing/stopSparkCluster.sh 1.6.0 2.3
# spark 1.5
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.5.2 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.5.2-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.5 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.5.2 2.3
# spark 1.4
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.4.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.4.1 2.3
# spark 1.3
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.3.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.3.1 2.3
# spark 1.2
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.2.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.2.1 2.3
# spark 1.1
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.1.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.1.1 2.3
- mvn $TEST_FLAG $PROFILE -B
after_failure:
- cat target/rat.txt
@ -77,7 +62,7 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
after_script:
-
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
notifications:
slack:

40
CONTRIBUTING.md
View file

@ -39,6 +39,29 @@ You can also use this small bookmarklet tool to fill your Pull Request fields au
javascript:(function() {var e = document.getElementById('pull_request_body');if (e) {e.value += '### What is this PR for?\nA few sentences describing the overall goals of the pull request\'s commits.\n\n### What type of PR is it?\n[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]\n\n### Todos\n* [ ] - Task\n\n### Is there a relevant Jira issue?\n\n### How should this be tested?\nOutline the steps to test the PR here.\n\n### Screenshots (if appropriate)\n\n### Questions:\n* Does the licenses files need update?\n* Is there breaking changes for older versions?\n* Does this needs documentation?';}})();
```
## Testing a Pull Request
You can also test and review a particular Pull Request. Here are two useful ways.
* Using a utility provided from Zeppelin.
```
dev/test_zeppelin_pr.py [# of PR]
```
For example, if you want to test `#513`, then the command will be:
```
dev/test_zeppelin_pr.py 513
```
* Another way is using [github/hub](https://github.com/github/hub).
```
hub checkout https://github.com/apache/incubator-zeppelin/pull/[# of PR]
```
The above two methods will help you test and review Pull Requests.
## Source Control Workflow
Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model.
@ -53,7 +76,6 @@ When a Pull Request is submitted, it is being merged or rejected by following re
* Committer can initiate lazy consensus ("Merge if there is no more discussion") and the code can be merged after certain time (normally 24 hours) when there is no review exists.
* Contributor can ping reviewers (including committer) by commenting 'Ready to review' or suitable indication.
## Becoming a Committer
The PPMC adds new committers from the active contributors, based on their contribution to Zeppelin. The qualifications for new committers include:
@ -68,7 +90,8 @@ Here are some things you will need to build and test Zeppelin.
### Software Configuration Management (SCM)
Zeppelin uses Git for its SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` you'll need git client installed in your development machine.
Zeppelin uses Git for its SCM system. `http://git.apache.org/incubator-zeppelin.git` you'll need git client installed in your development machine.
For write access, `https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git`
### Integrated Development Environment (IDE)
@ -114,26 +137,31 @@ To build the code, install
* Apache Maven
## Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
First of all, you need the Zeppelin source code. The official location for Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
### git access
Get the source code on your development machine using git.
```
git clone git@github.com:apache/incubator-zeppelin.git zeppelin
git clone http://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
```
git clone -b branch-0.1 git@github.com:apache/incubator-zeppelin.git zeppelin
git clone -b branch-0.1 http://git.apache.org/incubator-zeppelin.git zeppelin
```
or with write access
```
git clone https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git
```
### Fork repository
If you want not only build Zeppelin but also make change, then you need fork Zeppelin repository and make pull request.
If you want not only build Zeppelin but also make change, then you need fork Zeppelin github mirror repository (https://github.com/apache/incubator-zeppelin) and make pull request.
## Build

3
README.md
View file

@ -1,6 +1,6 @@
#Zeppelin
**Documentation:** [User Guide](http://zeppelin.incubator.apache.org/docs/index.html)<br/>
**Documentation:** [User Guide](http://zeppelin.incubator.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.incubator.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/incubator-zeppelin.png?branch=master)](https://travis-ci.org/apache/incubator-zeppelin) <br/>
**Contributing:** [Contribution Guide](https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md)<br/>
@ -45,6 +45,7 @@ sudo ln -s /usr/local/apache-maven-3.3.3/bin/mvn /usr/local/bin/mvn
_Notes:_
- Ensure node is installed by running `node --version`
- Ensure maven is running version 3.1.x or higher with `mvn -version`
- Configure maven to use more memory than usual by ```export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=1024m"```
### Build
If you want to build Zeppelin from the source, please first clone this repository, then:

43
SECURITY-README.md Normal file
View file

@ -0,0 +1,43 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Shiro Authentication
To connect to Zeppelin, users will be asked to enter their credentials. Once logged, a user has access to all notes including other users notes.
This a a first step toward full security as implemented by this pull request (https://github.com/apache/incubator-zeppelin/pull/53).
# Security setup
1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authcBasic" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections.
2. Secure the Websocket channel : Set to property "zeppelin.anonymous.allowed" to "false" in the file conf/zeppelin-site.xml. You can start by renaming conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
3. Start Zeppelin : bin/zeppelin.sh
4. point your browser to http://localhost:8080
5. Login using one of the user/password combinations defined in the conf/shiro.ini file.
# Implementation notes
## Vocabulary
username, owner and principal are used interchangeably to designate the currently authenticated user
## What are we securing ?
Zeppelin is basically a web application that spawn remote interpreters to run commands and return HTML fragments to be displayed on the user browser.
The scope of this PR is to require credentials to access Zeppelin. To achieve this, we use Apache Shiro.
## HTTP Endpoint security
Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info).
## Websocket security
Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach:
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above).

3
bin/zeppelin-daemon.sh
View file

@ -67,6 +67,9 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
fi
# Add jdbc connector jar
# ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar"
addJarInDir "${ZEPPELIN_HOME}"
addJarInDir "${ZEPPELIN_HOME}/lib"
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"

38
conf/shiro.ini Normal file
View file

@ -0,0 +1,38 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password1
user1 = password2
user2 = password3
# Sample LDAP configuration, for user Authentication, currently tested for single Realm
[main]
#ldapRealm = org.apache.shiro.realm.ldap.JndiLdapRealm
#ldapRealm.userDnTemplate = cn={0},cn=engg,ou=testdomain,dc=testdomain,dc=com
#ldapRealm.contextFactory.url = ldap://ldaphost:389
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
[urls]
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# To enfore security, comment the line below and uncomment the next one
/** = anon
#/** = authcBasic

8
conf/zeppelin-site.xml.template
View file

@ -105,7 +105,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
@ -180,5 +180,11 @@
<description>Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173</description>
</property>
<property>
<name>zeppelin.anonymous.allowed</name>
<value>true</value>
<description>Anonymous user allowed by default</description>
</property>
</configuration>

2
dev/create_release.sh
View file

@ -125,7 +125,7 @@ function make_binary_release() {
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
}
make_binary_release all "-Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark"
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark"
# remove non release files and dirs
rm -rf ${WORKING_DIR}/zeppelin

1
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -37,6 +37,7 @@
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 322 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 298 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_setting_1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 396 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_setting_2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 146 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_setting_3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 257 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_user.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 325 KiB

46
docs/development/writingzeppelininterpreter.md
View file

@ -32,10 +32,8 @@ All Interpreters in the same interpreter group are launched in a single, separat
### Make your own Interpreter
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include org.apache.zeppelin:zeppelin-interpreter:[VERSION] artifact in your build system.
Your interpreter name is derived from the static register method
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system.
Your interpreter name is derived from the static register method.
```
static {
@ -44,16 +42,15 @@ static {
```
The name will appear later in the interpreter name option box during the interpreter configuration process.
The name of the interpreter is what you later write to identify a paragraph which should be interpreted using this interpreter.
```
%MyInterpreterName
some interpreter spesific code...
some interpreter specific code...
```
### Install your interpreter binary
Once you have build your interpreter, you can place your interpreter under directory with all the dependencies.
Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies.
```
[ZEPPELIN_HOME]/interpreter/[INTERPRETER_NAME]/
@ -63,33 +60,34 @@ Once you have build your interpreter, you can place your interpreter under direc
To configure your interpreter you need to follow these steps:
1. create conf/zeppelin-site.xml by copying conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
1. Add your interpreter class name to the zeppelin.interpreters property in `conf/zeppelin-site.xml`.
2. Add your interpreter class name to the zeppelin.interpreters property in conf/zeppelin-site.xml
Property value is comma separated [INTERPRETER_CLASS_NAME]
for example,
Property value is comma separated [INTERPRETER\_CLASS\_NAME].
For example,
```
```
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter</value>
</property>
```
3. start zeppelin by running ```./bin/zeppelin-deamon start```
4. in the interpreter page, click the +Create button and configure your interpreter properties.
2. Add your interpreter to the [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `zeppelin-site.xml`.
3. Start Zeppelin by running `./bin/zeppelin-daemon.sh start`.
4. In the interpreter page, click the `+Create` button and configure your interpreter properties.
Now you are done and ready to use your interpreter.
Note that the interpreters shipped with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no zeppelin-site.xml.
Note that the interpreters shipped with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
### Use your interpreter
#### 0.5.0
Inside of a notebook, %[INTERPRETER_NAME] directive will call your interpreter.
Inside of a notebook, `%[INTERPRETER_NAME]` directive will call your interpreter.
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
for example
For example,
```
%myintp
@ -100,16 +98,14 @@ println(a)
<br />
#### 0.6.0 and later
Inside of a notebook, %[INTERPRETER\_GROUP].[INTERPRETER\_NAME] directive will call your interpreter.
Inside of a notebook, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter.
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. Omit [INTERPRETER\_NAME] selects first available interpreter in the [INTERPRETER\_GROUP].
Omit '[INTERPRETER\_GROUP]' will selects [INTERPRETER\_NAME] from default interpreter group.
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. If you omit [INTERPRETER\_NAME], then first available interpreter will be selected in the [INTERPRETER\_GROUP].
Likewise, if you skip [INTERPRETER\_GROUP], then [INTERPRETER\_NAME] will be chosen from default interpreter group.
For example, if you have two interpreter myintp1 and myintp2 in group mygrp,
you can call myintp1 like
For example, if you have two interpreter myintp1 and myintp2 in group mygrp, you can call myintp1 like
```
%mygrp.myintp1
@ -125,7 +121,7 @@ and you can call myintp2 like
codes for myintp2
```
If you omit your interpreter name, it'll selects first available interpreter in the group (myintp1)
If you omit your interpreter name, it'll select first available interpreter in the group ( myintp1 ).
```
%mygrp

2
docs/index.md
View file

@ -117,7 +117,7 @@ This way, you can easily embed it as an iframe inside of your website.</p>
<br />
### 100% Opensource
Apache Zeppelin (incubating) is Apache2 Licensed software. Please check out the [source repository](https://github.com/apache/incubator-zeppelin) and [How to contribute](./development/howtocontribute.html)
Apache Zeppelin (incubating) is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/incubator-zeppelin.git) and [How to contribute](./development/howtocontribute.html)
Zeppelin has a very active development community.
Join the [Mailing list](./community.html) and report issues on our [Issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN).

1062
docs/interpreter/cassandra.md
View file

File diff suppressed because it is too large Load diff

142
docs/interpreter/elasticsearch.md
View file

@ -6,12 +6,10 @@ group: manual
---
{% include JB/setup %}
## Elasticsearch Interpreter for Apache Zeppelin
[Elasticsearch](https://www.elastic.co/products/elasticsearch) is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements.
### 1. Configuration
<br/>
## Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
@ -31,7 +29,7 @@ group: manual
<tr>
<td>elasticsearch.port</td>
<td>9300</td>
<td>Connection port <b>(important: this is not the HTTP port, but the transport port)</b></td>
<td>Connection port <b>( Important: this is not the HTTP port, but the transport port )</b></td>
</tr>
<tr>
<td>elasticsearch.result.size</td>
@ -44,29 +42,20 @@ group: manual
![Interpreter configuration](../assets/themes/zeppelin/img/docs-img/elasticsearch-config.png)
</center>
> **Note #1 :** You can add more properties to configure the Elasticsearch client.
> Note #1: you can add more properties to configure the Elasticsearch client.
> Note #2: if you use Shield, you can add a property named `shield.user` with a value containing the name and the password (format: `username:password`). For more details about Shield configuration, consult the [Shield reference guide](https://www.elastic.co/guide/en/shield/current/_using_elasticsearch_java_clients_with_shield.html). Do not forget, to copy the shield client jar in the interpreter directory (`ZEPPELIN_HOME/interpreters/elasticsearch`).
<hr/>
### 2. Enabling the Elasticsearch Interpreter
> **Note #2 :** If you use Shield, you can add a property named `shield.user` with a value containing the name and the password ( format: `username:password` ). For more details about Shield configuration, consult the [Shield reference guide](https://www.elastic.co/guide/en/shield/current/_using_elasticsearch_java_clients_with_shield.html). Do not forget, to copy the shield client jar in the interpreter directory (`ZEPPELIN_HOME/interpreters/elasticsearch`).
## Enabling the Elasticsearch Interpreter
In a notebook, to enable the **Elasticsearch** interpreter, click the **Gear** icon and select **Elasticsearch**.
<hr/>
### 3. Using the Elasticsearch Interpreter
## Using the Elasticsearch Interpreter
In a paragraph, use `%elasticsearch` to select the Elasticsearch interpreter and then input all commands. To get the list of available commands, use `help`.
```bash
| %elasticsearch
| help
%elasticsearch
help
Elasticsearch interpreter:
General format: <command> /<indices>/<types>/<id> <option> <JSON>
- indices: list of indices separated by commas (depends on the command)
@ -82,84 +71,79 @@ Commands:
. same comments as for the search
- get /index/type/id
- delete /index/type/id
- index /ndex/type/id <json-formatted document>
- index /index/type/id <json-formatted document>
. the id can be omitted, elasticsearch will generate one
```
> Tip: use (CTRL + .) for completion
> **Tip :** Use ( Ctrl + . ) for autocompletion.
#### get
### Get
With the `get` command, you can find a document by id. The result is a JSON document.
```bash
| %elasticsearch
| get /index/type/id
%elasticsearch
get /index/type/id
```
Example:
![Elasticsearch - Get](../assets/themes/zeppelin/img/docs-img/elasticsearch-get.png)
#### search
### Search
With the `search` command, you can send a search query to Elasticsearch. There are two formats of query:
* You can provide a JSON-formatted query, that is exactly what you provide when you use the REST API of Elasticsearch.
* See [Elasticsearch search API reference document](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) for more details about the content of the search queries.
* You can also provide the content of a `query_string`
* You can also provide the content of a `query_string`.
* This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }`
* See [Elasticsearch query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) for more details about the content of such a query.
```bash
| %elasticsearch
| search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
%elasticsearch
search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
```
If you want to modify the size of the result set, you can add a line that is setting the size, before your search command.
```bash
| %elasticsearch
| size 50
| search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
%elasticsearch
size 50
search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
```
> A search query can also contain [aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html). If there is at least one aggregation, the result of the first aggregation is shown, otherwise, you get the search hits.
Examples:
* With a JSON query:
```bash
| %elasticsearch
| search / { "query": { "match_all": {} } }
|
| %elasticsearch
| search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
|
| %elasticsearch
| search /logs { "aggs": {
| "content_length_stats": {
| "extended_stats": {
| "field": "content_length"
| }
| }
| } }
%elasticsearch
search / { "query": { "match_all": { } } }
%elasticsearch
search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
%elasticsearch
search /logs { "aggs": {
"content_length_stats": {
"extended_stats": {
"field": "content_length"
}
}
} }
```
* With query_string elements:
```bash
| %elasticsearch
| search /logs request.method:GET AND status:200
|
| %elasticsearch
| search /logs (404 AND (POST OR DELETE))
%elasticsearch
search /logs request.method:GET AND status:200
%elasticsearch
search /logs (404 AND (POST OR DELETE))
```
> **Important**: a document in Elasticsearch is a JSON document, so it is hierarchical, not flat as a row in a SQL table.
> **Important** : a document in Elasticsearch is a JSON document, so it is hierarchical, not flat as a row in a SQL table.
For the Elastic interpreter, the result of a search query is flattened.
Suppose we have a JSON document:
@ -179,12 +163,10 @@ Suppose we have a JSON document:
The data will be flattened like this:
content_length | date | request.headers[0] | request.headers[1] | request.method | request.url | status
---------------|------|--------------------|--------------------|----------------|-------------|-------
1234 | 2015-12-08T21:03:13.588Z | Accept: \*.\* | Host: apache.org | GET | /zeppelin/4cd001cd-c517-4fa9-b8e5-a06b8f4056c4 | 403
Examples:
* With a table containing the results:
@ -205,13 +187,12 @@ Examples:
* With a query containing a multi-bucket aggregation:
![Elasticsearch - Search with aggregation (multi-bucket)](../assets/themes/zeppelin/img/docs-img/elasticsearch-agg-multi-bucket-pie.png)
#### count
### Count
With the `count` command, you can count documents available in some indices and types. You can also provide a query.
```bash
| %elasticsearch
| count /index1,index2,.../type1,type2,... <JSON document containing the query OR a query string>
%elasticsearch
count /index1,index2,.../type1,type2,... <JSON document containing the query OR a query string>
```
Examples:
@ -222,35 +203,30 @@ Examples:
* With a query:
![Elasticsearch - Count with query](../assets/themes/zeppelin/img/docs-img/elasticsearch-count-with-query.png)
#### index
### Index
With the `index` command, you can insert/update a document in Elasticsearch.
```bash
| %elasticsearch
| index /index/type/id <JSON document>
|
| %elasticsearch
| index /index/type <JSON document>
%elasticsearch
index /index/type/id <JSON document>
%elasticsearch
index /index/type <JSON document>
```
#### delete
### Delete
With the `delete` command, you can delete a document.
```bash
| %elasticsearch
| delete /index/type/id
%elasticsearch
delete /index/type/id
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```bash
| %elasticsearch
| size ${limit=10}
| search /index/type { "query": { "match_all": {} } }
%elasticsearch
size ${limit=10}
search /index/type { "query": { "match_all": { } } }
```

25
docs/interpreter/flink.md
View file

@ -6,15 +6,13 @@ group: manual
---
{% include JB/setup %}
## Flink interpreter for Apache Zeppelin
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing.
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing. Flink’s core is a streaming dataflow engine that provides data distribution, communication, and fault tolerance for distributed computations over data streams. Flink also builds batch processing on top of the streaming engine, overlaying native iteration support, managed memory, and program optimization.
### How to start local Flink cluster, to test the interpreter
## How to start local Flink cluster, to test the interpreter
Zeppelin comes with pre-configured flink-local interpreter, which starts Flink in a local mode on your machine, so you do not need to install anything.
### How to configure interpreter to point to Flink cluster
## How to configure interpreter to point to Flink cluster
At the "Interpreters" menu, you have to create a new Flink interpreter and provide next properties:
<table class="table-configuration">
@ -33,26 +31,19 @@ At the "Interpreters" menu, you have to create a new Flink interpreter and provi
<td>6123</td>
<td>port of running JobManager</td>
</tr>
<tr>
<td>xxx</td>
<td>yyy</td>
<td>anything else from [Flink Configuration](https://ci.apache.org/projects/flink/flink-docs-release-0.9/setup/config.html)</td>
</tr>
</table>
<br />
For more information about Flink configuration, you can find it [here](https://ci.apache.org/projects/flink/flink-docs-release-0.10/setup/config.html).
### How to test it's working
In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL05GTGFicy96ZXBwZWxpbi1ub3RlYm9va3MvbWFzdGVyL25vdGVib29rcy8yQVFFREs1UEMvbm90ZS5qc29u) is from [Till Rohrmann's presentation](http://www.slideshare.net/tillrohrmann/data-analysis-49806564) "Interactive data analysis with Apache Flink" for Apache Flink Meetup.
## How to test it's working
In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL05GTGFicy96ZXBwZWxpbi1ub3RlYm9va3MvbWFzdGVyL25vdGVib29rcy8yQVFFREs1UEMvbm90ZS5qc29u) is from Till Rohrmann's presentation [Interactive data analysis with Apache Flink](http://www.slideshare.net/tillrohrmann/data-analysis-49806564) for Apache Flink Meetup.
```
%sh
rm 10.txt.utf-8
wget http://www.gutenberg.org/ebooks/10.txt.utf-8
```
```
{% highlight scala %}
%flink
case class WordCount(word: String, frequency: Int)
val bible:DataSet[String] = env.readTextFile("10.txt.utf-8")
@ -65,4 +56,4 @@ val wordCounts = partialCounts.groupBy("word").reduce{
(left, right) => WordCount(left.word, left.frequency + right.frequency)
}
val result10 = wordCounts.first(10).collect()
```
{% endhighlight %}

60
docs/interpreter/geode.md
View file

@ -6,10 +6,7 @@ group: manual
---
{% include JB/setup %}
## Geode/Gemfire OQL Interpreter for Apache Zeppelin
<br/>
<table class="table-configuration">
<tr>
<th>Name</th>
@ -23,7 +20,6 @@ group: manual
</tr>
</table>
<br/>
This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Object Query Language (OQL)](http://geode-docs.cfapps.io/docs/developing/querying_basics/oql_compared_to_sql.html). With the OQL-based querying language:
[<img align="right" src="http://img.youtube.com/vi/zvzzA9GXu3Q/3.jpg" alt="zeppelin-view" hspace="10" width="200"></img>](https://www.youtube.com/watch?v=zvzzA9GXu3Q)
@ -38,7 +34,6 @@ This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Objec
This [Video Tutorial](https://www.youtube.com/watch?v=zvzzA9GXu3Q) illustrates some of the features provided by the `Geode Interpreter`.
### Create Interpreter
By default Zeppelin creates one `Geode/OQL` instance. You can remove it or create more instances.
Multiple Geode instances can be created, each configured to the same or different backend Geode cluster. But over time a `Notebook` can have only one Geode interpreter instance `bound`. That means you _cannot_ connect to different Geode clusters in the same `Notebook`. This is a known Zeppelin limitation.
@ -53,38 +48,35 @@ In the `Notebook` click on the `settings` icon in the top right corner. The sele
### Configuration
You can modify the configuration of the Geode from the `Interpreter` section. The Geode interpreter expresses the following properties:
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>geode.locator.host</td>
<td>The Geode Locator Host</td>
<td>localhost</td>
</tr>
<tr>
<td>geode.locator.port</td>
<td>The Geode Locator Port</td>
<td>10334</td>
</tr>
<tr>
<td>geode.max.result</td>
<td>Max number of OQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>geode.locator.host</td>
<td>The Geode Locator Host</td>
<td>localhost</td>
</tr>
<tr>
<td>geode.locator.port</td>
<td>The Geode Locator Port</td>
<td>10334</td>
</tr>
<tr>
<td>geode.max.result</td>
<td>Max number of OQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
### How to use
> *Tip 1: Use (CTRL + .) for OQL auto-completion.*
> *Tip 2: Always start the paragraphs with the full `%geode.oql` prefix tag! The short notation: `%geode` would still be able run the OQL queries but the syntax highlighting and the auto-completions will be disabled.*
#### Create / Destroy Regions
The OQL specification does not support [Geode Regions](https://cwiki.apache.org/confluence/display/GEODE/Index#Index-MainConceptsandComponents) mutation operations. To `create`/`destroy` regions one should use the [GFSH](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/chapter_overview.html) shell tool instead. In the following it is assumed that the GFSH is colocated with Zeppelin server.
```bash
@ -105,9 +97,7 @@ EOF
Above snippet re-creates two regions: `regionEmployee` and `regionCompany`. Note that you have to explicitly specify the locator host and port. The values should match those you have used in the Geode Interpreter configuration. Comprehensive list of [GFSH Commands by Functional Area](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/gfsh_quick_reference.html).
#### Basic OQL
#### Basic OQL
```sql
%geode.oql
SELECT count(*) FROM /regionEmployee
@ -144,12 +134,9 @@ Following query will return the EntrySet value as a Blob:
SELECT e.key, e.value FROM /regionEmployee.entrySet e
```
> Note: You can have multiple queries in the same paragraph but only the result from the first is displayed. [[1](https://issues.apache.org/jira/browse/ZEPPELIN-178)], [[2](https://issues.apache.org/jira/browse/ZEPPELIN-212)].
#### GFSH Commands From The Shell
Use the Shell Interpreter (`%sh`) to run OQL commands form the command line:
```bash
@ -159,7 +146,6 @@ gfsh -e "connect" -e "list members"
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your OQL queries. You can use both the `text input` and `select form` parameterization features
```sql

30
docs/interpreter/hive.md
View file

@ -6,12 +6,10 @@ group: manual
---
{% include JB/setup %}
## Hive Interpreter for Apache Zeppelin
The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL.
### Configuration
<br/>
<table class="table-configuration">
<tr>
<th>Property</th>
@ -31,49 +29,48 @@ group: manual
<tr>
<td>default.user</td>
<td></td>
<td><b>(Optional)</b>Username of the connection</td>
<td><b>( Optional ) </b>Username of the connection</td>
</tr>
<tr>
<td>default.password</td>
<td></td>
<td><b>(Optional)</b>Password of the connection</td>
<td><b>( Optional ) </b>Password of the connection</td>
</tr>
<tr>
<td>default.xxx</td>
<td></td>
<td><b>(Optional)</b>Other properties used by the driver</td>
<td><b>( Optional ) </b>Other properties used by the driver</td>
</tr>
<tr>
<td>${prefix}.driver</td>
<td></td>
<td>Driver class path of `%hive(${prefix})`</td>
<td>Driver class path of <code>%hive(${prefix})</code> </td>
</tr>
<tr>
<td>${prefix}.url</td>
<td></td>
<td>Url of `%hive(${prefix})`</td>
<td>Url of <code>%hive(${prefix})</code> </td>
</tr>
<tr>
<td>${prefix}.user</td>
<td></td>
<td><b>(Optional)</b>Username of the connection of `%hive(${prefix})`</td>
<td><b>( Optional ) </b>Username of the connection of <code>%hive(${prefix})</code> </td>
</tr>
<tr>
<td>${prefix}.password</td>
<td></td>
<td><b>(Optional)</b>Password of the connection of `%hive(${prefix})`</td>
<td><b>( Optional ) </b>Password of the connection of <code>%hive(${prefix})</code> </td>
</tr>
<tr>
<td>${prefix}.xxx</td>
<td></td>
<td><b>(Optional)</b>Other properties used by the driver of `%hive(${prefix})`</td>
<td><b>( Optional ) </b>Other properties used by the driver of <code>%hive(${prefix})</code> </td>
</tr>
</table>
This interpreter provides multiple configuration with ${prefix}. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
### How to use
This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
## How to use
Basically, you can use
```sql
@ -91,9 +88,8 @@ select * from my_table;
You can also run multiple queries up to 10 by default. Changing these settings is not implemented yet.
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```sql
%hive

119
docs/interpreter/ignite.md
View file

@ -18,55 +18,54 @@ You can use Zeppelin to retrieve distributed data from cache using Ignite SQL in
### Installing and Running Ignite example
In order to use Ignite interpreters, you may install Apache Ignite in some simple steps:
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zepplin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zepplin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
* Set examples directory path to Eclipse and select the pom.xml.
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
> **Tip. If you want to run Ignite examples on the cli not IDE, you can export executable Jar file from IDE. Then run it by using below command.**
```
$ nohup java -jar </path/to/your Jar file name>
```
### Configuring Ignite Interpreter
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
* Set examples directory path to Eclipse and select the pom.xml.
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
> **Tip. If you want to run Ignite examples on the cli not IDE, you can export executable Jar file from IDE. Then run it by using below command.**
```
$ nohup java -jar </path/to/your Jar file name>
```
### Configuring Ignite Interpreter
At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Zeppelin provides these properties for Ignite.
<table class="table-configuration">
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
</tr>
<tr>
<td>ignite.addresses</td>
<td>127.0.0.1:47500..47509</td>
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/v1.2/docs/cluster-config) section for more details.</td>
<td>ignite.addresses</td>
<td>127.0.0.1:47500..47509</td>
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/v1.2/docs/cluster-config) section for more details.</td>
</tr>
<tr>
<td>ignite.clientMode</td>
<td>true</td>
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/v1.2/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
<td>ignite.clientMode</td>
<td>true</td>
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/v1.2/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
</tr>
<tr>
<td>ignite.config.url</td>
<td></td>
<td>Configuration URL. Overrides all other settings.</td>
</tr
<tr>
<td>ignite.jdbc.url</td>
<td>jdbc:ignite:cfg://default-ignite-jdbc.xml</td>
<td>Ignite JDBC connection URL.</td>
</tr>
<tr>
<td>ignite.peerClassLoadingEnabled</td>
<td>true</td>
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/v1.2/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
<td>ignite.config.url</td>
<td></td>
<td>Configuration URL. Overrides all other settings.</td>
</tr>
</table>
<tr>
<td>ignite.jdbc.url</td>
<td>jdbc:ignite:cfg://default-ignite-jdbc.xml</td>
<td>Ignite JDBC connection URL.</td>
</tr>
<tr>
<td>ignite.peerClassLoadingEnabled</td>
<td>true</td>
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/v1.2/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
</tr>
</table>
![Configuration of Ignite Interpreter](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-setting.png)
@ -82,35 +81,33 @@ In order to execute SQL query, use ` %ignite.ignitesql ` prefix. <br>
Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ).
For example, you can select top 10 words in the words cache using the following query
```
%ignite.ignitesql
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
```
![IgniteSql on Zeppelin](../assets/themes/zeppelin/img/docs-img/ignite-sql-example.png)
```
%ignite.ignitesql
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
```
![IgniteSql on Zeppelin](../assets/themes/zeppelin/img/docs-img/ignite-sql-example.png)
As long as your Ignite version and Zeppelin Ignite version is same, you can also use scala code. Please check the Zeppelin Ignite version before you download your own Ignite.
```
%ignite
import org.apache.ignite._
import org.apache.ignite.cache.affinity._
import org.apache.ignite.cache.query._
import org.apache.ignite.configuration._
```
%ignite
import org.apache.ignite._
import org.apache.ignite.cache.affinity._
import org.apache.ignite.cache.query._
import org.apache.ignite.configuration._
import scala.collection.JavaConversions._
import scala.collection.JavaConversions._
val cache: IgniteCache[AffinityUuid, String] = ignite.cache("words")
val cache: IgniteCache[AffinityUuid, String] = ignite.cache("words")
val qry = new SqlFieldsQuery("select avg(cnt), min(cnt), max(cnt) from (select count(_val) as cnt from String group by _val)", true)
val qry = new SqlFieldsQuery("select avg(cnt), min(cnt), max(cnt) from (select count(_val) as cnt from String group by _val)", true)
val res = cache.query(qry).getAll()
val res = cache.query(qry).getAll()
collectionAsScalaIterable(res).foreach(println _)
```
![Using Scala Code](../assets/themes/zeppelin/img/docs-img/ignite-scala-example.png)
collectionAsScalaIterable(res).foreach(println _)
```
![Using Scala Code](../assets/themes/zeppelin/img/docs-img/ignite-scala-example.png)
Apache Ignite also provides a guide docs for Zeppelin ["Ignite with Apache Zeppelin"](https://apacheignite.readme.io/docs/data-analysis-with-apache-zeppelin)

196
docs/interpreter/lens.md
View file

@ -16,69 +16,70 @@ group: manual
### Installing and Running Lens
In order to use Lens interpreters, you may install Apache Lens in some simple steps:
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
3. Now, you can start lens server (or stop).
```
./bin/lens-ctl start (or stop)
```
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
3. Now, you can start lens server (or stop).
```
./bin/lens-ctl start (or stop)
```
### Configuring Lens Interpreter
At the "Interpreters" menu, you can to edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
<table class="table-configuration">
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
</tr>
<tr>
<td>lens.client.dbname</td>
<td>default</td>
<td>The database schema name</td>
<td>lens.client.dbname</td>
<td>default</td>
<td>The database schema name</td>
</tr>
<tr>
<td>lens.query.enable.persistent.resultset</td>
<td>false</td>
<td>Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions</td>
<td>lens.query.enable.persistent.resultset</td>
<td>false</td>
<td>Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions</td>
</tr>
<tr>
<td>lens.server.base.url</td>
<td>http://hostname:port/lensapi</td>
<td>The base url for the lens server. you have to edit "hostname" and "port" that you may use(ex. http://0.0.0.0:9999/lensapi)</td>
<td>lens.server.base.url</td>
<td>http://hostname:port/lensapi</td>
<td>The base url for the lens server. you have to edit "hostname" and "port" that you may use(ex. http://0.0.0.0:9999/lensapi)</td>
</tr>
<tr>
<td>lens.session.cluster.user </td>
<td>default</td>
<td>Hadoop cluster username</td>
<td>lens.session.cluster.user </td>
<td>default</td>
<td>Hadoop cluster username</td>
</tr>
<tr>
<td>zeppelin.lens.maxResult</td>
<td>1000</td>
<td>Max number of rows to display</td>
<td>zeppelin.lens.maxResult</td>
<td>1000</td>
<td>Max number of rows to display</td>
</tr>
<tr>
<td>zeppelin.lens.maxThreads</td>
<td>10</td>
<td>If concurrency is true then how many threads?</td>
<td>zeppelin.lens.maxThreads</td>
<td>10</td>
<td>If concurrency is true then how many threads?</td>
</tr>
<tr>
<td>zeppelin.lens.run.concurrent</td>
<td>true</td>
<td>Run concurrent Lens Sessions</td>
<td>zeppelin.lens.run.concurrent</td>
<td>true</td>
<td>Run concurrent Lens Sessions</td>
</tr>
<tr>
<td>xxx</td>
<td>yyy</td>
<td>anything else from [Configuring lens server](https://lens.apache.org/admin/config-server.html)</td>
<td>xxx</td>
<td>yyy</td>
<td>anything else from [Configuring lens server](https://lens.apache.org/admin/config-server.html)</td>
</tr>
</table>
</table>
![Apache Lens Interpreter Setting](../assets/themes/zeppelin/img/docs-img/lens-interpreter-setting.png)
### Interpreter Bindging for Zeppelin Notebook
After configuring Lens interpreter, create your own notebook, then you can bind interpreters like below image.
After configuring Lens interpreter, create your own notebook, then you can bind interpreters like below image.
![Zeppelin Notebook Interpreter Biding](../assets/themes/zeppelin/img/docs-img/lens-interpreter-binding.png)
For more interpreter binding information see [here](http://zeppelin.incubator.apache.org/docs/manual/interpreters.html).
@ -90,84 +91,79 @@ As you can see in this video, they are using Lens Client Shell(./bin/lens-cli.sh
<li> Create and Use(Switch) Databases.
```
create database newDb
```
```
use newDb
```
```
create database newDb
```
```
use newDb
```
<li> Create Storage.
```
create storage your/path/to/lens/client/examples/resources/db-storage.xml
```
```
create storage your/path/to/lens/client/examples/resources/db-storage.xml
```
<li> Create Dimensions, Show fields and join-chains of them.
```
create dimension your/path/to/lens/client/examples/resources/customer.xml
```
```
dimension show fields customer
```
```
dimension show joinchains customer
```
```
create dimension your/path/to/lens/client/examples/resources/customer.xml
```
```
dimension show fields customer
```
```
dimension show joinchains customer
```
<li> Create Caches, Show fields and join-chains of them.
```
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
```
```
cube show fields sales
```
```
cube show joinchains sales
```
```
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
```
```
cube show fields sales
```
```
cube show joinchains sales
```
<li> Create Dimtables and Fact.
```
create dimtable your/path/to/lens/client/examples/resources/customer_table.xml
```
```
create fact your/path/to/lens/client/examples/resources/sales-raw-fact.xml
```
```
create dimtable your/path/to/lens/client/examples/resources/customer_table.xml
```
```
create fact your/path/to/lens/client/examples/resources/sales-raw-fact.xml
```
<li> Add partitions to Dimtable and Fact.
```
dimtable add single-partition --dimtable_name customer_table --storage_name local --path your/path/to/lens/client/examples/resources/customer-local-part.xml
```
```
fact add partitions --fact_name sales_raw_fact --storage_name local --path your/path/to/lens/client/examples/resources/sales-raw-local-parts.xml
```
```
dimtable add single-partition --dimtable_name customer_table --storage_name local --path your/path/to/lens/client/examples/resources/customer-local-part.xml
```
```
fact add partitions --fact_name sales_raw_fact --storage_name local --path your/path/to/lens/client/examples/resources/sales-raw-local-parts.xml
```
<li> Now, you can run queries on cubes.
```
query execute cube select customer_city_name, product_details.description, product_details.category, product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-11-00', '2015-04-13-00')
```
![Lens Query Result](../assets/themes/zeppelin/img/docs-img/lens-result.png)
```
query execute cube select customer_city_name, product_details.description, product_details.category, product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-11-00', '2015-04-13-00')
```
![Lens Query Result](../assets/themes/zeppelin/img/docs-img/lens-result.png)
These are just examples that provided in advance by Lens. If you want to explore whole tutorials of Lens, see the [tutorial video](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
### Lens UI Service
Lens also provides web UI service. Once the server starts up, you can open the service on http://serverhost:19999/index.html and browse. You may also check the structure that you made and use query easily here.
![Lens UI Servive](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png)
![Lens UI Servive](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png)

9
docs/interpreter/markdown.md
View file

@ -10,13 +10,14 @@ group: manual
### Overview
[Markdown](http://daringfireball.net/projects/markdown/) is a plain text formatting syntax designed so that it can be converted to HTML.
Zeppelin uses markdown4j, for more examples and extension support checkout [markdown4j](https://code.google.com/p/markdown4j/)
In Zeppelin notebook you can use ``` %md ``` in the beginning of a paragraph to invoke the Markdown interpreter to generate static html from Markdown plain text.
Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/).
In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to invoke the Markdown interpreter and generate static html from Markdown plain text.
In Zeppelin, Markdown interpreter is enabled by default.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="600px" />
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="60%" />
### Example
The following example demonstrates the basic usage of Markdown in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="800px" />
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />

81
docs/interpreter/postgresql.md
View file

@ -6,10 +6,7 @@ group: manual
---
{% include JB/setup %}
## PostgreSQL, HAWQ Interpreter for Apache Zeppelin
<br/>
<table class="table-configuration">
<tr>
<th>Name</th>
@ -23,7 +20,6 @@ group: manual
</tr>
</table>
<br/>
[<img align="right" src="http://img.youtube.com/vi/wqXXQhJ5Uk8/0.jpg" alt="zeppelin-view" hspace="10" width="250"></img>](https://www.youtube.com/watch?v=wqXXQhJ5Uk8)
This interpreter seamlessly supports the following SQL data processing engines:
@ -32,11 +28,9 @@ This interpreter seamlessly supports the following SQL data processing engines:
* [Apache HAWQ](http://pivotal.io/big-data/pivotal-hawq) - Powerful [Open Source](https://wiki.apache.org/incubator/HAWQProposal) SQL-On-Hadoop engine.
* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL.
This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`.
### Create Interpreter
By default Zeppelin creates one `PSQL` instance. You can remove it or create new instances.
Multiple PSQL instances can be created, each configured to the same or different backend databases. But over time a `Notebook` can have only one PSQL interpreter instance `bound`. That means you _cannot_ connect to different databases in the same `Notebook`. This is a known Zeppelin limitation.
@ -51,47 +45,45 @@ In the `Notebook` click on the `settings` icon in the top right corner. The sele
### Configuration
You can modify the configuration of the PSQL from the `Interpreter` section. The PSQL interpreter expenses the following properties:
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>postgresql.url</td>
<td>JDBC URL to connect to </td>
<td>jdbc:postgresql://localhost:5432</td>
</tr>
<tr>
<td>postgresql.user</td>
<td>JDBC user name</td>
<td>gpadmin</td>
</tr>
<tr>
<td>postgresql.password</td>
<td>JDBC password</td>
<td></td>
</tr>
<tr>
<td>postgresql.driver.name</td>
<td>JDBC driver name. In this version the driver name is fixed and should not be changed</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>postgresql.max.result</td>
<td>Max number of SQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>postgresql.url</td>
<td>JDBC URL to connect to </td>
<td>jdbc:postgresql://localhost:5432</td>
</tr>
<tr>
<td>postgresql.user</td>
<td>JDBC user name</td>
<td>gpadmin</td>
</tr>
<tr>
<td>postgresql.password</td>
<td>JDBC password</td>
<td></td>
</tr>
<tr>
<td>postgresql.driver.name</td>
<td>JDBC driver name. In this version the driver name is fixed and should not be changed</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>postgresql.max.result</td>
<td>Max number of SQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
### How to use
```
Tip: Use (CTRL + .) for SQL auto-completion.
```
#### DDL and SQL commands
#### DDL and SQL commands
Start the paragraphs with the full `%psql.sql` prefix tag! The short notation: `%psql` would still be able run the queries but the syntax highlighting and the auto-completions will be disabled.
You can use the standard CREATE / DROP / INSERT commands to create or modify the data model:
@ -121,7 +113,6 @@ select * from mytable;
```
#### PSQL command line tools
Use the Shell Interpreter (`%sh`) to access the command line [PSQL](http://www.postgresql.org/docs/9.4/static/app-psql.html) interactively:
```bash
@ -131,6 +122,7 @@ psql -h phd3.localdomain -U gpadmin -p 5432 <<EOF
\q
EOF
```
This will produce output like this:
```
@ -146,7 +138,6 @@ This will produce output like this:
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
@ -157,8 +148,8 @@ GROUP BY ${group_by=product_id,product_id|product_name|customer_id|store_id}
ORDER BY count ${order=DESC,DESC|ASC}
LIMIT ${limit=10};
```
#### Example HAWQ PXF/HDFS Tables
#### Example HAWQ PXF/HDFS Tables
Create HAWQ external table that read data from tab-separated-value data in HDFS.
```sql
@ -168,11 +159,13 @@ CREATE EXTERNAL TABLE retail_demo.payment_methods_pxf (
payment_method_code character varying(20)
) LOCATION ('pxf://${NAME_NODE_HOST}:50070/retail_demo/payment_methods.tsv.gz?profile=HdfsTextSimple') FORMAT 'TEXT' (DELIMITER = E'\t');
```
And retrieve content
```sql
%psql.sql
select * from retail_demo.payment_methods_pxf
```
### Auto-completion
The PSQL Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. In addition to the SQL keyword the interpreter provides suggestions for the Schema, Table, Column names as well.

17
docs/interpreter/scalding.md
View file

@ -6,7 +6,6 @@ group: manual
---
{% include JB/setup %}
## Scalding Interpreter for Apache Zeppelin
[Scalding](https://github.com/twitter/scalding) is an open source Scala library for writing MapReduce jobs.
@ -18,20 +17,20 @@ mvn clean package -Pscalding -DskipTests
```
### Enabling the Scalding Interpreter
In a notebook, to enable the **Scalding** interpreter, click on the **Gear** icon,select **Scalding**, and hit **Save**.
<center>
![Interpreter Binding](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterBinding.png)
![Interpreter Selection](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterSelection.png)
</center>
<center>
![Interpreter Binding](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterBinding.png)
![Interpreter Selection](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterSelection.png)
</center>
### Configuring the Interpreter
Zeppelin comes with a pre-configured Scalding interpreter in local mode, so you do not need to install anything.
### Testing the Interpreter
In example, by using the [Alice in Wonderland](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) tutorial, we will count words (of course!), and plot a graph of the top 10 words in the book.
```
@ -75,4 +74,4 @@ If you click on the icon for the pie chart, you should be able to see a chart li
### Current Status & Future Work
The current implementation of the Scalding interpreter does not support canceling jobs, or fine-grained progress updates.
The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).
The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).

121
docs/interpreter/spark.md
View file

@ -7,8 +7,7 @@ group: manual
{% include JB/setup %}
## Spark Interpreter
## Spark Interpreter for Apache Zeppelin
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
Spark Interpreter group, which consisted of 4 interpreters.
@ -40,18 +39,11 @@ Spark Interpreter group, which consisted of 4 interpreters.
</tr>
</table>
## Configuration
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
<br /><br />
### Configuration
<hr />
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need following two simple steps.
#### 1. export SPARK_HOME
In **conf/zeppelin-env.sh**, export SPARK_HOME environment variable with your Spark installation path.
### 1. Export SPARK_HOME
In **conf/zeppelin-env.sh**, export `SPARK_HOME` environment variable with your Spark installation path.
for example
@ -66,10 +58,7 @@ export HADOOP_CONF_DIR=/usr/lib/hadoop
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"
```
<br />
#### 2. set master in Interpreter menu.
### 2. Set master in Interpreter menu
After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type.
for example,
@ -79,32 +68,22 @@ for example,
* **yarn-client** in Yarn client mode
* **mesos://host:5050** in Mesos cluster
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. ( Zeppelin 0.5.5-incubating release works up to Spark 1.5.2 )
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
<br />
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuild Zeppelin in this way. (Zeppelin 0.5.5-incubating release works up to Spark 1.5.1)
Note that without exporting SPARK_HOME, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
<br /> <br />
### SparkContext, SQLContext, ZeppelinContext
<hr />
## SparkContext, SQLContext, ZeppelinContext
SparkContext, SQLContext, ZeppelinContext are automatically created and exposed as variable names 'sc', 'sqlContext' and 'z', respectively, both in scala and python environments.
Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
> Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
<a name="dependencyloading"> </a>
<br />
<br />
### Dependency Management
<hr />
There are two ways to load external library in spark interpreter. First is using Zeppelin's %dep interpreter and second is loading Spark properties.
#### 1. Dynamic Dependency Loading via %dep interpreter
## Dependency Management
There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties.
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using %dep interpreter.
### 1. Dynamic Dependency Loading via %dep interpreter
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
* Load libraries recursively from Maven repository
* Load libraries from local filesystem
@ -112,7 +91,7 @@ When your code requires external library, instead of doing download/copy/restart
* Automatically add libraries to SparkCluster (You can turn off)
Dep interpreter leverages scala environment. So you can write any Scala code here.
Note that %dep interpreter should be used before %spark, %pyspark, %sql.
Note that `%dep` interpreter should be used before `%spark`, `%pyspark`, `%sql`.
Here's usages.
@ -150,9 +129,7 @@ z.load("groupId:artifactId:version").exclude("groupId:*")
z.load("groupId:artifactId:version").local()
```
<br />
#### 2. Loading Spark Properties
### 2. Loading Spark Properties
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
<table class="table-configuration">
@ -181,9 +158,8 @@ Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit`
<td>Comma-separated list of files to be placed in the working directory of each executor.</td>
</tr>
</table>
Note that adding jar to pyspark is only availabe via %dep interpreter at the moment
> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
<br/>
Here are few examples:
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
@ -196,41 +172,43 @@ Here are few examples:
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
<br />
<br />
### ZeppelinContext
<hr />
## ZeppelinContext
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
<br />
#### Object exchange
### Object Exchange
ZeppelinContext extends map and it's shared between scala, python environment.
So you can put some object from scala and read it from python, vise versa.
Put object from scala
<div class="codetabs">
<div data-lang="scala" markdown="1">
```scala
{% highlight scala %}
// Put object from scala
%spark
val myObject = ...
z.put("objName", myObject)
```
{% endhighlight %}
Get object from python
</div>
<div data-lang="python" markdown="1">
```python
%python
{% highlight python %}
# Get object from python
%pyspark
myObject = z.get("objName")
```
{% endhighlight %}
</div>
</div>
<br />
#### Form creation
### Form Creation
ZeppelinContext provides functions for creating forms.
In scala and python environments, you can create forms programmatically.
<div class="codetabs">
<div data-lang="scala" markdown="1">
```scala
{% highlight scala %}
%spark
/* Create text input form */
z.input("formName")
@ -245,7 +223,30 @@ z.select("formName", Seq(("option1", "option1DisplayName"),
/* Create select form with default value*/
z.select("formName", "option1", Seq(("option1", "option1DisplayName"),
("option2", "option2DisplayName")))
```
{% endhighlight %}
</div>
<div data-lang="python" markdown="1">
{% highlight python %}
%pyspark
# Create text input form
z.input("formName")
# Create text input form with default value
z.input("formName", "defaultValue")
# Create select form
z.select("formName", [("option1", "option1DisplayName"),
("option2", "option2DisplayName")])
# Create select form with default value
z.select("formName", [("option1", "option1DisplayName"),
("option2", "option2DisplayName")], "option1")
{% endhighlight %}
</div>
</div>
In sql environment, you can create form in simple template.

124
docs/manual/dynamicinterpreterload.md Normal file
View file

@ -0,0 +1,124 @@
---
layout: page
title: "Dynamic Interpreter Loading"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## Dynamic Interpreter Loading using REST API
Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal).
Before we start, if you are not familiar with the concept of **Zeppelin interpreter**, you can check out [Overview of Zeppelin interpreter](../manual/interpreters.html) first.
<br/>
## Overview
In the past, Zeppelin was loading interpreter binaries from `/interpreter/[interpreter_name]` directory. They were configured by `zeppelin.interpreters` property in `conf/zeppelin-site.xml` or `ZEPPELIN_INTERPRETERS` env variables in `conf/zeppelin-env.sh`. They were loaded on Zeppelin server startup and stayed alive until the server was stopped.
In order to simplify using 3rd party interpreters, we changed this way to **dynamically** load interpreters from **Maven Repository** using **REST API**. Hopefully, the picture below will help you to understand the process.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin_user.png" height="85%" width="85%"></center>
## Load & Unload Interpreters Using REST API
### 1. Load
You can **load** interpreters located in Maven repository using REST API, like this:
( Maybe, you are unfamiliar with `[interpreter_group_name]` or `[interpreter_name]`. If so, please checkout [Interpreters in Zeppelin](../manual/interpreter.html) again. )
```
http://[zeppelin-server]:[zeppelin-port]/api/interpreter/load/[interpreter_group_name]/[interpreter_name]
```
The Restful method will be <code>**POST**</code>. And the parameters you need are:
1. **Artifact:** Maven artifact ( groupId:artifactId:version )
2. **Class Name:** Package name + Interpreter class name
3. **Repository ( optional ):** Additional maven repository address
For example, if you want to load `markdown` interpreter to your Zeppelin, the parameters and URL you need may look like:
```
http://127.0.0.1:8080/api/interpreter/load/md/markdown
```
```
{
"artifact": "org.apache.zeppelin:zeppelin-markdown:0.6.0-incubating-SNAPSHOT",
"className": "org.apache.zeppelin.markdown.Markdown",
"repository": {
"url": "http://dl.bintray.com/spark-packages/maven",
"snapshot": false
}
}
```
The meaning of each parameters is:
1. **Artifact**
- groupId: org.apache.zeppelin
- artifactId: zeppelin-markdown
- version: 0.6.0-incubating-SNAPSHOT
2. **Class Name**
- Package Name: org.apache.zeppelin
- Interpreter Class Name: markdown.Markdown
3. **Repository ( optional )**
- Url: http://dl.bintray.com/spark-packages/maven
- Snapshot: false
> <b>Please note: </b>The interpreters you downloaded need to be **reload**, when your Zeppelin server is down.
### 2. Unload
If you want to **unload** the interpreters using REST API,
```
http://[zeppelin-server]:[zeppelin-port]/api/interpreter/unload/[interpreter_group_name]/[interpreter_name]
```
In this case, the Restful method will be <code>**DELETE**</code>.
<br/>
## What is the next step after Loading ?
### Q1. Where is the location of interpreters you downloaded ?
Actually, the answer about this question is in the above picture. Once the REST API is called, the `.jar` files of interpreters you get are saved under `ZEPPELIN_HOME/local-repo` first. Then, they will be copied to `ZEPPELIN_HOME/interpreter` directory. So, please checkout your `ZEPPELIN_HOME/interpreter`.
### Q2. Then, how can I use this interpreter ?
After loading an interpreter, you can use it by creating and configuring it in Zeppelin's **Interpreter tab**.
Oh, you don't need to restart your Zeppelin server. Because it is **Dynamic Loading**, you can configure and load it **at runtime** !
1. After Zeppelin server up, browse Zeppelin home and click **Interpreter tab**.
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_1.png" height="85%" width="85%"></center>
2. At the **Interpreter** section, click **+Create** button.
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_2.png" height="85%" width="85%"></center>
3. Then, you can verify the interpreter list that you loaded.
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_3.png" height="85%" width="85%"></center>
4. After choosing an interpreter, you can configure and use it. Don't forget to save it.
5. Create a new notebook in the **Notebook** section, then you can bind the interpreters from your interpreter list. Just drag and drop !
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_binding_1.png" height="85%" width="85%"></center>
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_binding_2.png" height="85%" width="85%"></center>
6. At last, you can use your interpreter !
If you want to get the specific information about respective interpreters, please checkout each interpreter documentation.

52
docs/manual/interpreters.md
View file

@ -19,46 +19,40 @@ limitations under the License.
-->
{% include JB/setup %}
## Interpreters in Zeppelin
In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin.
The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin.
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, Hive, Markdown, Shell and so on.
## Interpreters in zeppelin
## What is Zeppelin interpreter?
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use scala code in Zeppelin, you need `%spark` interpreter.
This section explain the role of Interpreters, interpreters group and interpreters settings in Zeppelin.
Zeppelin interpreter concept allows any language/data-processing-backend to be plugged into Zeppelin.
Currently Zeppelin supports many interpreters such as Scala(with Apache Spark), Python(with Apache Spark), SparkSQL, Hive, Markdown and Shell.
### What is zeppelin interpreter?
Zeppelin Interpreter is the plug-in which enable zeppelin user to use a specific language/data-processing-backend. For example to use scala code in Zeppelin, you need ```spark``` interpreter.
When you click on the ```+Create``` button in the interpreter page the interpreter drop-down list box will present all the available interpreters on your server.
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
### What is zeppelin interpreter setting?
Zeppelin interpreter setting is the configuration of a given interpreter on zeppelin server. For example, the properties requried for hive JDBC interpreter to connect to the Hive server.
## What is Zeppelin Interpreter Setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
### What is zeppelin interpreter group?
Every Interpreter belongs to an InterpreterGroup. InterpreterGroup is a unit of start/stop interpreter.
By default, every interpreter belong to a single group but the group might contain more interpreters. For example, spark interpreter group include spark support, pySpark,
## What is Zeppelin Interpreter Group?
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, spark interpreter group is including Spark support, pySpark,
SparkSQL and the dependency loader.
Technically, Zeppelin interpreters from the same group are running in the same JVM.
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
Interpreters belong to a single group a registered together and all of their properties are listed in the interpreter setting.
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
### Programming langages for interpreter
If the interpreter uses a specific programming language (like Scala, Python, SQL), it is generally a good idea to add syntax highlighting support for that to the notebook paragraph editor.
To check out the list of languages supported, see the mode-*.js files under zeppelin-web/bower_components/ace-builds/src-noconflict or from github https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict
To add a new set of syntax highlighting,
1. add the mode-*.js file to zeppelin-web/bower.json (when built, zeppelin-web/src/index.html will be changed automatically)
2. add to the list of `editorMode` in zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js - it follows the pattern 'ace/mode/x' where x is the name
3. add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` in zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
## Programming Languages for Interpreter
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.

73
docs/rest-api/rest-interpreter.md
View file

@ -22,28 +22,29 @@ limitations under the License.
## Zeppelin REST API
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
All REST API are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
All REST API are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON view such as
[JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
[JSON View](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
If you work with zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.incubator.apache.org/community.html).
<br />
### Interpreter REST API list
## Interpreter REST API List
The role of registered interpreters, settings and interpreters group is described [here](../manual/interpreters.html)
The role of registered interpreters, settings and interpreters group are described in [here](../manual/interpreters.html).
### 1. List of Registered Interpreters & Interpreter Settings
<table class="table-configuration">
<col width="200">
<tr>
<th>List registered interpreters</th>
<th>List of registered interpreters</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```GET``` method return all the registered interpreters available on the server.</td>
<td>This ```GET``` method returns all the registered interpreters available on the server.</td>
</tr>
<tr>
<td>URL</td>
@ -54,12 +55,11 @@ limitations under the License.
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>
{
@ -113,12 +113,12 @@ limitations under the License.
<table class="table-configuration">
<col width="200">
<tr>
<th>List interpreters settings</th>
<th>List of interpreters settings</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```GET``` method return all the interpreters settings registered on the server.</td>
<td>This ```GET``` method returns all the interpreters settings registered on the server.</td>
</tr>
<tr>
<td>URL</td>
@ -129,12 +129,11 @@ limitations under the License.
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>
{
@ -182,7 +181,8 @@ limitations under the License.
</table>
<br/>
### 2. Create an Interpreter Setting
<table class="table-configuration">
<col width="200">
<tr>
@ -202,12 +202,11 @@ limitations under the License.
<td>201</td>
</tr>
<tr>
<td> Fail code</td>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON input
</td>
<td>Sample JSON input</td>
<td>
<pre>
{
@ -227,8 +226,7 @@ limitations under the License.
</td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>
{
@ -256,7 +254,8 @@ limitations under the License.
<br/>
### 3. Update an Interpreter Setting
<table class="table-configuration">
<col width="200">
<tr>
@ -276,12 +275,11 @@ limitations under the License.
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON input
</td>
<td>Sample JSON input</td>
<td>
<pre>
{
@ -301,8 +299,7 @@ limitations under the License.
</td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>
{
@ -330,7 +327,8 @@ limitations under the License.
<br/>
### 4. Delete an Interpreter Setting
<table class="table-configuration">
<col width="200">
<tr>
@ -354,17 +352,17 @@ limitations under the License.
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>{"status":"OK"}</pre>
<code>{"status":"OK"}</code>
</td>
</tr>
</table>
<br/>
### 5. Restart an Interpreter
<table class="table-configuration">
<col width="200">
<tr>
@ -373,7 +371,7 @@ limitations under the License.
</tr>
<tr>
<td>Description</td>
<td>This ```PUT``` method restart the given interpreter id.</td>
<td>This ```PUT``` method restarts the given interpreter id.</td>
</tr>
<tr>
<td>URL</td>
@ -384,14 +382,13 @@ limitations under the License.
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>Sample JSON response</td>
<td>
<pre>{"status":"OK"}</pre>
<code>{"status":"OK"}</code>
</td>
</tr>
</table>

28
docs/tutorial/tutorial.md
View file

@ -17,20 +17,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
### Zeppelin Tutorial
## Zeppelin Tutorial
We will assume you have Zeppelin installed already. If that's not the case, see [Install](../install/install.html).
This tutorial walks you through some of the fundamental Zeppelin concepts. We will assume you have already installed Zeppelin. If not, please see [here](../install/install.html) first.
Zeppelin's current main backend processing engine is [Apache Spark](https://spark.apache.org). If you're new to the system, you might want to start by getting an idea of how it processes data to get the most out of Zeppelin.
Current main backend processing engine of Zeppelin is [Apache Spark](https://spark.apache.org). If you're new to this system, you might want to start by getting an idea of how it processes data to get the most out of Zeppelin.
<br />
### Tutorial with Local File
## Tutorial with Local File
#### Data Refine
### 1. Data Refine
Before you start Zeppelin tutorial, you will need to download [bank.zip](http://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip).
First, to transform data from csv format into RDD of `Bank` objects, run following script. This will also remove header using `filter` function.
First, to transform csv format data into RDD of `Bank` objects, run following script. This will also remove header using `filter` function.
```scala
@ -38,7 +38,7 @@ val bankText = sc.textFile("yourPath/bank/bank-full.csv")
case class Bank(age:Integer, job:String, marital : String, education : String, balance : Integer)
// split each line, filter out header (starts with "age"), and map it into Bank case class
// split each line, filter out header (starts with "age"), and map it into Bank case class
val bank = bankText.map(s=>s.split(";")).filter(s=>s(0)!="\"age\"").map(
s=>Bank(s(0).toInt,
s(1).replaceAll("\"", ""),
@ -52,8 +52,7 @@ val bank = bankText.map(s=>s.split(";")).filter(s=>s(0)!="\"age\"").map(
bank.toDF().registerTempTable("bank")
```
<br />
#### Data Retrieval
### 2. Data Retrieval
Suppose we want to see age distribution from `bank`. To do this, run:
@ -74,9 +73,9 @@ Now we want to see age distribution with certain marital status and add combo bo
```
<br />
### Tutorial with Streaming Data
## Tutorial with Streaming Data
#### Data Refine
### 1. Data Refine
Since this tutorial is based on Twitter's sample tweet stream, you must configure authentication with a Twitter account. To do this, take a look at [Twitter Credential Setup](https://databricks-training.s3.amazonaws.com/realtime-processing-with-spark-streaming.html#twitter-credential-setup). After you get API keys, you should fill out credential related values(`apiKey`, `apiSecret`, `accessToken`, `accessTokenSecret`) with your API keys on following script.
@ -136,12 +135,11 @@ twt.print
ssc.start()
```
<br />
#### Data Retrieval
### 2. Data Retrieval
For each following script, every time you click run button you will see different result since it is based on real-time data.
Let's begin by extracting maximum 10 tweets which contain the word "girl".
Let's begin by extracting maximum 10 tweets which contain the word **girl**.
```sql
%sql select * from tweets where text like '%girl%' limit 10
@ -154,7 +152,7 @@ This time suppose we want to see how many tweets have been created per sec durin
```
You can make user-defined function and use it in Spark SQL. Let's try it by making function named `sentiment`. This function will return one of the three attitudes(positive, negative, neutral) towards the parameter.
You can make user-defined function and use it in Spark SQL. Let's try it by making function named `sentiment`. This function will return one of the three attitudes( positive, negative, neutral ) towards the parameter.
```scala
def sentiment(s:String) : String = {

4
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java
View file

@ -141,6 +141,10 @@ public class ElasticsearchInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) {
logger.info("Run Elasticsearch command '" + cmd + "'");
if (StringUtils.isEmpty(cmd) || StringUtils.isEmpty(cmd.trim())) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
}
int currentResultSize = resultSize;

10
elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java
View file

@ -198,4 +198,14 @@ public class ElasticsearchInterpreterTest {
assertEquals("11", res.message());
}
@Test
public void testMisc() {
InterpreterResult res = interpreter.interpret(null, null);
assertEquals(Code.SUCCESS, res.code());
res = interpreter.interpret(" \n \n ", null);
assertEquals(Code.SUCCESS, res.code());
}
}

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class FlinkInterpreterTest {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null);
context = new InterpreterContext(null, null, null, null, null, null, null, null, null);
}
@AfterClass

12
hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java
View file

@ -79,9 +79,9 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null)).message().contains("SCHEMA_NAME"));
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null,null)).message().contains("SCHEMA_NAME"));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null)).message());
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
}
@Test
@ -101,7 +101,7 @@ public class HiveInterpreterTest {
t.open();
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null)).message());
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
}
@Test
@ -117,13 +117,13 @@ public class HiveInterpreterTest {
t.open();
InterpreterResult interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
t.getConnection("default").close();
interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@ -139,7 +139,7 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null);
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null);
//simple select test
InterpreterResult result = t.interpret("select * from test_table", interpreterContext);

1
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -211,6 +211,7 @@ public class IgniteInterpreter extends Interpreter {
initEx = null;
} catch (Exception e) {
logger.error("Error in IgniteInterpreter while getIgnite: " , e);
initEx = e;
}
}

2
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java
View file

@ -154,6 +154,7 @@ public class IgniteSqlInterpreter extends Interpreter {
}
}
} catch (Exception e) {
logger.error("Exception in IgniteSqlInterpreter while InterpreterResult interpret: ", e);
return IgniteInterpreterUtils.buildErrorResult(e);
} finally {
curStmt = null;
@ -169,6 +170,7 @@ public class IgniteSqlInterpreter extends Interpreter {
curStmt.cancel();
} catch (SQLException e) {
// No-op.
logger.info("No-op while cancel in IgniteSqlInterpreter", e);
} finally {
curStmt = null;
}

7
ignite/src/main/resources/default-ignite-jdbc.xml
View file

@ -33,13 +33,6 @@
<property name="peerClassLoadingEnabled" value="true"/>
<property name="marshaller">
<bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
<!-- Set to false to allow non-serializable objects in examples, default is true. -->
<property name="requireSerializable" value="false"/>
</bean>
</property>
<!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;

3
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -44,7 +44,7 @@ public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;
@ -60,7 +60,6 @@ public class IgniteSqlInterpreterTest {
IgniteConfiguration cfg = new IgniteConfiguration();
cfg.setDiscoverySpi(discoSpi);
cfg.setPeerClassLoadingEnabled(true);
cfg.setMarshaller(new OptimizedMarshaller());
cfg.setGridName("test");

169
jdbc/pom.xml Normal file
View file

@ -0,0 +1,169 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-jdbc</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: JDBC interpreter</name>
<url>http://www.apache.org</url>
<properties>
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>2.12.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.190</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mockrunner</groupId>
<artifactId>mockrunner-jdbc</artifactId>
<version>1.0.8</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

413
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java Normal file
View file

@ -0,0 +1,413 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
/**
* JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ,
* GreenplumDB, MariaDB, MySQL, Postgres and Redshit.
*
* <ul>
* <li>{@code default.url} - JDBC URL to connect to.</li>
* <li>{@code default.user} - JDBC user name..</li>
* <li>{@code default.password} - JDBC password..</li>
* <li>{@code default.driver.name} - JDBC driver name.</li>
* <li>{@code common.max.result} - Max number of SQL result to display.</li>
* </ul>
*
* <p>
* How to use: <br/>
* {@code %jdbc.sql} <br/>
* {@code
* SELECT store_id, count(*)
* FROM retail_demo.order_lineitems_pxf
* GROUP BY store_id;
* }
* </p>
*
*/
public class JDBCInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(JDBCInterpreter.class);
static final String COMMON_KEY = "common";
static final String MAX_LINE_KEY = "max_count";
static final String MAX_LINE_DEFAULT = "1000";
static final String DEFAULT_KEY = "default";
static final String DRIVER_KEY = "driver";
static final String URL_KEY = "url";
static final String USER_KEY = "user";
static final String PASSWORD_KEY = "password";
static final String DOT = ".";
private static final char WHITESPACE = ' ';
private static final char NEWLINE = '\n';
private static final char TAB = '\t';
private static final String TABLE_MAGIC_TAG = "%table ";
private static final String EXPLAIN_PREDICATE = "EXPLAIN ";
private static final String UPDATE_COUNT_HEADER = "Update Count";
static final String COMMON_MAX_LINE = COMMON_KEY + DOT + MAX_LINE_KEY;
static final String DEFAULT_DRIVER = DEFAULT_KEY + DOT + DRIVER_KEY;
static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
static final String EMPTY_COLUMN_VALUE = "";
private final HashMap<String, Properties> propertiesMap;
private final Map<String, Statement> paragraphIdStatementMap;
private final Map<String, ArrayList<Connection>> propertyKeyUnusedConnectionListMap;
private final Map<String, Connection> paragraphIdConnectionMap;
static {
Interpreter.register(
"sql",
"jdbc",
JDBCInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(DEFAULT_URL, "jdbc:postgresql://localhost:5432/", "The URL for JDBC.")
.add(DEFAULT_USER, "gpadmin", "The JDBC user name")
.add(DEFAULT_PASSWORD, "",
"The JDBC user password")
.add(DEFAULT_DRIVER, "org.postgresql.Driver", "JDBC Driver Name")
.add(COMMON_MAX_LINE, MAX_LINE_DEFAULT,
"Max number of SQL result to display.").build());
}
public JDBCInterpreter(Properties property) {
super(property);
propertiesMap = new HashMap<>();
propertyKeyUnusedConnectionListMap = new HashMap<>();
paragraphIdStatementMap = new HashMap<>();
paragraphIdConnectionMap = new HashMap<>();
}
public HashMap<String, Properties> getPropertiesMap() {
return propertiesMap;
}
@Override
public void open() {
for (String propertyKey : property.stringPropertyNames()) {
logger.debug("propertyKey: {}", propertyKey);
String[] keyValue = propertyKey.split("\\.", 2);
if (2 == keyValue.length) {
logger.info("key: {}, value: {}", keyValue[0], keyValue[1]);
Properties prefixProperties;
if (propertiesMap.containsKey(keyValue[0])) {
prefixProperties = propertiesMap.get(keyValue[0]);
} else {
prefixProperties = new Properties();
propertiesMap.put(keyValue[0], prefixProperties);
}
prefixProperties.put(keyValue[1], property.getProperty(propertyKey));
}
}
Set<String> removeKeySet = new HashSet<>();
for (String key : propertiesMap.keySet()) {
if (!COMMON_KEY.equals(key)) {
Properties properties = propertiesMap.get(key);
if (!properties.containsKey(DRIVER_KEY) || !properties.containsKey(URL_KEY)) {
logger.error("{} will be ignored. {}.{} and {}.{} is mandatory.",
key, DRIVER_KEY, key, key, URL_KEY);
removeKeySet.add(key);
}
}
}
for (String key : removeKeySet) {
propertiesMap.remove(key);
}
logger.debug("propertiesMap: {}", propertiesMap);
}
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
Connection connection = null;
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
if (0 != connectionList.size()) {
connection = propertyKeyUnusedConnectionListMap.get(propertyKey).remove(0);
if (null != connection && connection.isClosed()) {
connection.close();
connection = null;
}
}
}
if (null == connection) {
Properties properties = propertiesMap.get(propertyKey);
logger.info(properties.getProperty(DRIVER_KEY));
Class.forName(properties.getProperty(DRIVER_KEY));
String url = properties.getProperty(URL_KEY);
String user = properties.getProperty(USER_KEY);
String password = properties.getProperty(PASSWORD_KEY);
if (null != user && null != password) {
connection = DriverManager.getConnection(url, user, password);
} else {
connection = DriverManager.getConnection(url, properties);
}
}
return connection;
}
public Statement getStatement(String propertyKey, String paragraphId)
throws SQLException, ClassNotFoundException {
Connection connection;
if (paragraphIdConnectionMap.containsKey(paragraphId)) {
connection = paragraphIdConnectionMap.get(paragraphId);
} else {
connection = getConnection(propertyKey);
}
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
connection = getConnection(propertyKey);
statement = connection.createStatement();
}
paragraphIdConnectionMap.put(paragraphId, connection);
paragraphIdStatementMap.put(paragraphId, statement);
return statement;
}
private boolean isStatementClosed(Statement statement) {
try {
return statement.isClosed();
} catch (Throwable t) {
logger.debug("{} doesn't support isClosed method", statement);
return false;
}
}
@Override
public void close() {
try {
for (List<Connection> connectionList : propertyKeyUnusedConnectionListMap.values()) {
for (Connection c : connectionList) {
c.close();
}
}
for (Statement statement : paragraphIdStatementMap.values()) {
statement.close();
}
paragraphIdStatementMap.clear();
for (Connection connection : paragraphIdConnectionMap.values()) {
connection.close();
}
paragraphIdConnectionMap.clear();
} catch (SQLException e) {
logger.error("Error while closing...", e);
}
}
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
String paragraphId = interpreterContext.getParagraphId();
try {
Statement statement = getStatement(propertyKey, paragraphId);
statement.setMaxRows(getMaxResult());
StringBuilder msg = null;
boolean isTableType = false;
if (containsIgnoreCase(sql, EXPLAIN_PREDICATE)) {
msg = new StringBuilder();
} else {
msg = new StringBuilder(TABLE_MAGIC_TAG);
isTableType = true;
}
ResultSet resultSet = null;
try {
boolean isResultSetAvailable = statement.execute(sql);
if (isResultSetAvailable) {
resultSet = statement.getResultSet();
ResultSetMetaData md = resultSet.getMetaData();
for (int i = 1; i < md.getColumnCount() + 1; i++) {
if (i > 1) {
msg.append(TAB);
}
msg.append(replaceReservedChars(isTableType, md.getColumnName(i)));
}
msg.append(NEWLINE);
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
msg.append(replaceReservedChars(isTableType, resultSet.getString(i)));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
}
msg.append(NEWLINE);
displayRowCount++;
}
} else {
// Response contains either an update count or there are no results.
int updateCount = statement.getUpdateCount();
msg.append(UPDATE_COUNT_HEADER).append(NEWLINE);
msg.append(updateCount).append(NEWLINE);
}
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
statement.close();
} finally {
statement = null;
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
} catch (SQLException ex) {
logger.error("Cannot run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
} catch (ClassNotFoundException e) {
logger.error("Cannot run " + sql, e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
/**
* For %table response replace Tab and Newline characters from the content.
*/
private String replaceReservedChars(boolean isTableResponseType, String str) {
if (str == null) {
return EMPTY_COLUMN_VALUE;
}
return (!isTableResponseType) ? str : str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE);
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run SQL command '{}'", cmd);
String propertyKey = getPropertyKey(cmd);
if (null != propertyKey) {
cmd = cmd.substring(propertyKey.length() + 2);
} else {
propertyKey = DEFAULT_KEY;
}
cmd = cmd.trim();
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
return executeSql(propertyKey, cmd, contextInterpreter);
}
@Override
public void cancel(InterpreterContext context) {
logger.info("Cancel current query statement.");
String paragraphId = context.getParagraphId();
try {
paragraphIdStatementMap.get(paragraphId).cancel();
} catch (SQLException e) {
logger.error("Error while cancelling...", e);
}
}
public String getPropertyKey(String cmd) {
int firstLineIndex = cmd.indexOf("\n");
if (-1 == firstLineIndex) {
firstLineIndex = cmd.length();
}
int configStartIndex = cmd.indexOf("(");
int configLastIndex = cmd.indexOf(")");
if (configStartIndex != -1 && configLastIndex != -1
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
return cmd.substring(configStartIndex + 1, configLastIndex);
}
return null;
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
JDBCInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
public int getMaxResult() {
return Integer.valueOf(
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
}
}

125
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java Normal file
View file

@ -0,0 +1,125 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.jdbc;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_KEY;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.jdbc.JDBCInterpreter;
import org.junit.Before;
import org.junit.Test;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
/**
* JDBC interpreter unit tests
*/
public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
static String jdbcConnection;
private static String getJdbcConnection() throws IOException {
if(null == jdbcConnection) {
Path tmpDir = Files.createTempDirectory("h2-test-");
tmpDir.toFile().deleteOnExit();
jdbcConnection = format("jdbc:h2:%s", tmpDir);
}
return jdbcConnection;
}
@Before
public void setUp() throws Exception {
Class.forName("org.h2.Driver");
Connection connection = DriverManager.getConnection(getJdbcConnection());
Statement statement = connection.createStatement();
statement.execute(
"DROP TABLE IF EXISTS test_table; " +
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
statement.execute(
"insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name');"
);
}
@Test
public void testDefaultProperties() throws SQLException {
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(new Properties());
assertEquals("org.postgresql.Driver", jdbcInterpreter.getProperty(DEFAULT_DRIVER));
assertEquals("jdbc:postgresql://localhost:5432/", jdbcInterpreter.getProperty(DEFAULT_URL));
assertEquals("gpadmin", jdbcInterpreter.getProperty(DEFAULT_USER));
assertEquals("", jdbcInterpreter.getProperty(DEFAULT_PASSWORD));
assertEquals("1000", jdbcInterpreter.getProperty(COMMON_MAX_LINE));
}
@Test
public void testSelectQuery() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@Test
public void testSelectQueryMaxResult() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message());
}
}

3
lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java
View file

@ -128,7 +128,7 @@ public class LensInterpreter extends Interpreter {
s_logger.info("LensInterpreter created");
}
catch (Exception e) {
e.printStackTrace();
s_logger.error(e.toString(), e);
s_logger.error("unable to create lens interpreter", e);
}
}
@ -375,6 +375,7 @@ public class LensInterpreter extends Interpreter {
closeShell(s_paraToQH.get(context.getParagraphId()).getShell());
} catch (Exception e) {
// ignore
s_logger.info("Exception in LensInterpreter while cancel finally, ignore", e);
}
s_paraToQH.remove(context.getParagraphId());
closeShell(shell);

7
lens/src/main/java/org/apache/zeppelin/lens/LensJLineShellComponent.java
View file

@ -17,6 +17,9 @@ package org.apache.zeppelin.lens;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactoryUtils;
import org.springframework.beans.factory.InitializingBean;
@ -56,6 +59,8 @@ public class LensJLineShellComponent extends JLineShell
private ExecutionStrategy executionStrategy = new LensSimpleExecutionStrategy();
private SimpleParser parser = new SimpleParser();
private static final Logger LOGGER = LoggerFactory.getLogger(LensJLineShellComponent.class);
public SimpleParser getSimpleParser() {
return parser;
}
@ -123,7 +128,7 @@ public class LensJLineShellComponent extends JLineShell
try {
shellThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
LOGGER.error(e.toString(), e);
}
}

4
markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java
View file

@ -29,12 +29,15 @@ import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.markdown4j.Markdown4jProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Markdown interpreter for Zeppelin.
*/
public class Markdown extends Interpreter {
private Markdown4jProcessor md;
static final Logger LOGGER = LoggerFactory.getLogger(Markdown.class);
static {
Interpreter.register("md", Markdown.class.getName());
@ -58,6 +61,7 @@ public class Markdown extends Interpreter {
try {
html = md.process(st);
} catch (IOException | java.lang.RuntimeException e) {
LOGGER.error("Exception in Markdown while interpret ", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
return new InterpreterResult(Code.SUCCESS, "%html " + html);

6
phoenix/pom.xml
View file

@ -45,6 +45,12 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
<version>1.1.9</version>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>

13
pom.xml
View file

@ -94,6 +94,7 @@
<module>hive</module>
<module>phoenix</module>
<module>postgresql</module>
<module>jdbc</module>
<module>tajo</module>
<module>flink</module>
<module>ignite</module>
@ -208,6 +209,18 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
<!-- Apache Shiro -->
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-core</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-web</artifactId>
<version>1.2.3</version>
</dependency>
</dependencies>
</dependencyManagement>

1
postgresql/src/main/java/org/apache/zeppelin/postgresql/PostgreSqlInterpreter.java
View file

@ -297,6 +297,7 @@ public class PostgreSqlInterpreter extends Interpreter {
try {
currentStatement.cancel();
} catch (SQLException ex) {
logger.error("SQLException in PostgreSqlInterpreter while cancel ", ex);
} finally {
currentStatement = null;
}

2
scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java
View file

@ -65,7 +65,7 @@ public class ScaldingInterpreterTest {
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), null);
}
@After

3
spark-dependencies/pom.xml
View file

@ -272,14 +272,12 @@
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-twitter_2.10</artifactId>
@ -292,7 +290,6 @@
<version>${spark.version}</version>
</dependency>
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>

129
spark/pom.xml
View file

@ -38,64 +38,8 @@
<spark.version>1.4.1</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<hadoop.version>2.3.0</hadoop.version>
<py4j.version>0.8.2.1</py4j.version>
</properties>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<profile>
<id>spark-1.1</id>
<properties>
<spark.version>1.1.1</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.2</id>
<properties>
<spark.version>1.2.1</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.3</id>
<properties>
<spark.version>1.3.1</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.4</id>
<properties>
<spark.version>1.4.1</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.5</id>
<properties>
<spark.version>1.5.2</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.6</id>
<properties>
<spark.version>1.6.0</spark.version>
<py4j.version>0.9</py4j.version>
</properties>
</profile>
</profiles>
<dependencies>
<dependency>
@ -247,51 +191,6 @@
<version>1.3</version>
</dependency>
<!-- to compile and test code.
Runtime dependency is provided by either spark-dependencies submodule or SPARK_HOME
-->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_2.10</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_2.10</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
@ -326,12 +225,6 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
<version>${py4j.version}</version>
</dependency>
<!--TEST-->
<dependency>
@ -346,6 +239,28 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.datanucleus</groupId>
<artifactId>datanucleus-core</artifactId>
<version>3.2.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.datanucleus</groupId>
<artifactId>datanucleus-api-jdo</artifactId>
<version>3.2.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.datanucleus</groupId>
<artifactId>datanucleus-rdbms</artifactId>
<version>3.2.9</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>

40
spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java
View file

@ -40,7 +40,9 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.resolution.ArtifactResolutionException;
import org.sonatype.aether.resolution.DependencyResolutionException;
@ -67,7 +69,9 @@ public class DepInterpreter extends Interpreter {
"spark",
DepInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.dep.localrepo", "local-repo", "local repository for dependency loader")
.add("zeppelin.dep.localrepo",
getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"),
"local repository for dependency loader")
.add("zeppelin.dep.additionalRemoteRepository",
"spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
"A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.")
@ -77,18 +81,39 @@ public class DepInterpreter extends Interpreter {
private SparkIMain intp;
private ByteArrayOutputStream out;
private DependencyContext depc;
private SparkDependencyContext depc;
private SparkJLineCompletion completor;
private SparkILoop interpreter;
static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class);
public DepInterpreter(Properties property) {
super(property);
}
public DependencyContext getDependencyContext() {
public SparkDependencyContext getDependencyContext() {
return depc;
}
public static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
return envValue;
}
}
if (propertyName != null && !propertyName.isEmpty()) {
String propValue = System.getProperty(propertyName);
if (propValue != null) {
return propValue;
}
}
return defaultValue;
}
@Override
public void close() {
@ -149,16 +174,16 @@ public class DepInterpreter extends Interpreter {
intp.setContextClassLoader();
intp.initializeSynchronous();
depc = new DependencyContext(getProperty("zeppelin.dep.localrepo"),
depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
completor = new SparkJLineCompletion(intp);
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getValue("_binder");
binder.put("depc", depc);
intp.interpret("@transient val z = "
+ "_binder.get(\"depc\").asInstanceOf[org.apache.zeppelin.spark.dep.DependencyContext]");
+ "_binder.get(\"depc\")"
+ ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]");
}
@ -195,6 +220,7 @@ public class DepInterpreter extends Interpreter {
depc.fetch();
} catch (MalformedURLException | DependencyResolutionException
| ArtifactResolutionException e) {
LOGGER.error("Exception in DepInterpreter while interpret ", e);
return new InterpreterResult(Code.ERROR, e.toString());
}

40
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -54,7 +54,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -73,8 +73,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
private GatewayServer gatewayServer;
private DefaultExecutor executor;
private int port;
private ByteArrayOutputStream outputStream;
private ByteArrayOutputStream errStream;
private SparkOutputStream outputStream;
private BufferedWriter ins;
private PipedInputStream in;
private ByteArrayOutputStream input;
@ -127,7 +126,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
URL [] urls = new URL[0];
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
List<URL> urlList = new LinkedList<URL>();
@ -173,7 +172,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
cmd.addArgument(Integer.toString(port), false);
cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
executor = new DefaultExecutor();
outputStream = new ByteArrayOutputStream();
outputStream = new SparkOutputStream();
PipedOutputStream ps = new PipedOutputStream();
in = null;
try {
@ -274,7 +273,6 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
statementError = error;
statementFinishedNotifier.notify();
}
}
boolean pythonScriptInitialized = false;
@ -287,6 +285,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public void appendOutput(String message) throws IOException {
outputStream.getInterpreterOutput().write(message);
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
@ -300,7 +302,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
+ outputStream.toString());
}
outputStream.reset();
outputStream.setInterpreterOutput(context.out);
synchronized (pythonScriptInitializeNotifier) {
long startTime = System.currentTimeMillis();
@ -314,15 +316,24 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
String errorMessage = "";
try {
context.out.flush();
errorMessage = new String(context.out.toByteArray());
} catch (IOException e) {
throw new InterpreterException(e);
}
if (pythonscriptRunning == false) {
// python script failed to initialize and terminated
return new InterpreterResult(Code.ERROR, "failed to start pyspark"
+ outputStream.toString());
+ errorMessage);
}
if (pythonScriptInitialized == false) {
// timeout. didn't get initialized message
return new InterpreterResult(Code.ERROR, "pyspark is not responding "
+ outputStream.toString());
+ errorMessage);
}
if (!sparkInterpreter.getSparkVersion().isPysparkSupported()) {
@ -352,7 +363,14 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
if (statementError) {
return new InterpreterResult(Code.ERROR, statementOutput);
} else {
return new InterpreterResult(Code.SUCCESS, statementOutput);
try {
context.out.flush();
} catch (IOException e) {
throw new InterpreterException(e);
}
return new InterpreterResult(Code.SUCCESS);
}
}
@ -389,8 +407,6 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return new LinkedList<String>();
}
outputStream.reset();
pythonInterpretRequest = new PythonInterpretRequest(completionCommand, "");
statementOutput = null;

54
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -17,9 +17,7 @@
package org.apache.zeppelin.spark;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
@ -41,7 +39,6 @@ import org.apache.spark.repl.SparkJLineCompletion;
import org.apache.spark.scheduler.ActiveJob;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.Pool;
import org.apache.spark.scheduler.SparkListener;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
@ -55,8 +52,8 @@ import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -82,7 +79,7 @@ import scala.tools.nsc.settings.MutableSettings.PathSetting;
*
*/
public class SparkInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
static {
Interpreter.register(
@ -115,9 +112,9 @@ public class SparkInterpreter extends Interpreter {
private SparkILoop interpreter;
private SparkIMain intp;
private SparkContext sc;
private ByteArrayOutputStream out;
private SparkOutputStream out;
private SQLContext sqlc;
private DependencyResolver dep;
private SparkDependencyResolver dep;
private SparkJLineCompletion completor;
private JobProgressListener sparkListener;
@ -129,7 +126,7 @@ public class SparkInterpreter extends Interpreter {
public SparkInterpreter(Properties property) {
super(property);
out = new ByteArrayOutputStream();
out = new SparkOutputStream();
}
public SparkInterpreter(Properties property, SparkContext sc) {
@ -186,7 +183,7 @@ public class SparkInterpreter extends Interpreter {
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
e.printStackTrace();
logger.error(e.toString(), e);
return null;
}
return pl;
@ -222,9 +219,9 @@ public class SparkInterpreter extends Interpreter {
return sqlc;
}
public DependencyResolver getDependencyResolver() {
public SparkDependencyResolver getDependencyResolver() {
if (dep == null) {
dep = new DependencyResolver(intp,
dep = new SparkDependencyResolver(intp,
sc,
getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
@ -335,6 +332,10 @@ public class SparkInterpreter extends Interpreter {
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
}
// Distributes needed libraries to workers.
if (getProperty("master").equals("yarn-client")) {
conf.set("spark.yarn.isPython", "true");
}
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
@ -423,7 +424,7 @@ public class SparkInterpreter extends Interpreter {
// add dependency from DepInterpreter
DepInterpreter depInterpreter = getDepInterpreter();
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
if (files != null) {
@ -448,10 +449,9 @@ public class SparkInterpreter extends Interpreter {
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
PrintStream printStream = new PrintStream(out);
/* spark interpreter */
this.interpreter = new SparkILoop(null, new PrintWriter(out));
interpreter.settings_$eq(settings);
interpreter.createInterpreter();
@ -477,7 +477,7 @@ public class SparkInterpreter extends Interpreter {
dep = getDependencyResolver();
z = new ZeppelinContext(sc, sqlc, null, dep, printStream,
z = new ZeppelinContext(sc, sqlc, null, dep,
Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
@ -485,7 +485,6 @@ public class SparkInterpreter extends Interpreter {
binder.put("sc", sc);
binder.put("sqlc", sqlc);
binder.put("z", z);
binder.put("out", printStream);
intp.interpret("@transient val z = "
+ "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]");
@ -532,7 +531,7 @@ public class SparkInterpreter extends Interpreter {
// add jar
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFilesDist();
if (files != null) {
@ -671,13 +670,13 @@ public class SparkInterpreter extends Interpreter {
synchronized (this) {
z.setGui(context.getGui());
sc.setJobGroup(getJobGroup(context), "Zeppelin", false);
InterpreterResult r = interpretInput(lines);
InterpreterResult r = interpretInput(lines, context);
sc.clearJobGroup();
return r;
}
}
public InterpreterResult interpretInput(String[] lines) {
public InterpreterResult interpretInput(String[] lines, InterpreterContext context) {
SparkEnv.set(env);
// add print("") to make sure not finishing with comment
@ -688,8 +687,9 @@ public class SparkInterpreter extends Interpreter {
}
linesToRun[lines.length] = "print(\"\")";
Console.setOut((java.io.PrintStream) binder.get("out"));
out.reset();
Console.setOut(context.out);
out.setInterpreterOutput(context.out);
context.out.clear();
Code r = null;
String incomplete = "";
@ -709,6 +709,7 @@ public class SparkInterpreter extends Interpreter {
res = intp.interpret(incomplete + s);
} catch (Exception e) {
sc.clearJobGroup();
out.setInterpreterOutput(null);
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
@ -717,7 +718,8 @@ public class SparkInterpreter extends Interpreter {
if (r == Code.ERROR) {
sc.clearJobGroup();
return new InterpreterResult(r, out.toString());
out.setInterpreterOutput(null);
return new InterpreterResult(r, "");
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
@ -726,9 +728,13 @@ public class SparkInterpreter extends Interpreter {
}
if (r == Code.INCOMPLETE) {
sc.clearJobGroup();
out.setInterpreterOutput(null);
return new InterpreterResult(r, "Incomplete expression");
} else {
return new InterpreterResult(r, out.toString());
sc.clearJobGroup();
out.setInterpreterOutput(null);
return new InterpreterResult(Code.SUCCESS);
}
}

75
spark/src/main/java/org/apache/zeppelin/spark/SparkOutputStream.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import java.io.IOException;
import java.io.OutputStream;
/**
* InterpreterOutput can be attached / detached.
*/
public class SparkOutputStream extends OutputStream {
InterpreterOutput interpreterOutput;
public SparkOutputStream() {
}
public InterpreterOutput getInterpreterOutput() {
return interpreterOutput;
}
public void setInterpreterOutput(InterpreterOutput interpreterOutput) {
this.interpreterOutput = interpreterOutput;
}
@Override
public void write(int b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte [] b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte [] b, int offset, int len) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b, offset, len);
}
}
@Override
public void close() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.close();
}
}
@Override
public void flush() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.flush();
}
}
}

24
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -21,6 +21,7 @@ import static scala.collection.JavaConversions.asJavaCollection;
import static scala.collection.JavaConversions.asJavaIterable;
import static scala.collection.JavaConversions.collectionAsScalaIterable;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -43,7 +44,7 @@ import org.apache.zeppelin.display.Input.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import scala.Tuple2;
import scala.Unit;
@ -53,20 +54,18 @@ import scala.collection.Iterable;
* Spark context for zeppelin.
*/
public class ZeppelinContext extends HashMap<String, Object> {
private DependencyResolver dep;
private PrintStream out;
private SparkDependencyResolver dep;
private InterpreterContext interpreterContext;
private int maxResult;
public ZeppelinContext(SparkContext sc, SQLContext sql,
InterpreterContext interpreterContext,
DependencyResolver dep, PrintStream printStream,
SparkDependencyResolver dep,
int maxResult) {
this.sc = sc;
this.sqlContext = sql;
this.interpreterContext = interpreterContext;
this.dep = dep;
this.out = printStream;
this.maxResult = maxResult;
}
@ -220,7 +219,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
paramOptions[i++] = new ParamOption(valueAndDisplayValue._1(), valueAndDisplayValue._2());
}
return gui.select(name, "", paramOptions);
return gui.select(name, defaultValue, paramOptions);
}
public void setGui(GUI o) {
@ -273,10 +272,15 @@ public class ZeppelinContext extends HashMap<String, Object> {
throw new InterpreterException("Can not road DataFrame/SchemaRDD class");
}
if (cls.isInstance(o)) {
out.print(showDF(sc, interpreterContext, o, maxResult));
} else {
out.print(o.toString());
try {
if (cls.isInstance(o)) {
interpreterContext.out.write(showDF(sc, interpreterContext, o, maxResult));
} else {
interpreterContext.out.write(o.toString());
}
} catch (IOException e) {
throw new InterpreterException(e);
}
}

181
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java Normal file
View file

@ -0,0 +1,181 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
import java.io.File;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.util.List;
import org.apache.zeppelin.dep.Booter;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.Repository;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.resolution.ArtifactResolutionException;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
import org.sonatype.aether.resolution.DependencyResolutionException;
import org.sonatype.aether.util.artifact.DefaultArtifact;
import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
/**
*
*/
public class SparkDependencyContext {
List<Dependency> dependencies = new LinkedList<Dependency>();
List<Repository> repositories = new LinkedList<Repository>();
List<File> files = new LinkedList<File>();
List<File> filesDist = new LinkedList<File>();
private RepositorySystem system = Booter.newRepositorySystem();
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
private RemoteRepository mavenLocal = Booter.newLocalRepository();
private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>();
public SparkDependencyContext(String localRepoPath, String additionalRemoteRepository) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
addRepoFromProperty(additionalRemoteRepository);
}
public Dependency load(String lib) {
Dependency dep = new Dependency(lib);
if (dependencies.contains(dep)) {
dependencies.remove(dep);
}
dependencies.add(dep);
return dep;
}
public Repository addRepo(String name) {
Repository rep = new Repository(name);
repositories.add(rep);
return rep;
}
public void reset() {
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
files = new LinkedList<File>();
filesDist = new LinkedList<File>();
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
for (String repo : repos) {
String[] parts = repo.split(",");
if (parts.length == 3) {
String id = parts[0].trim();
String url = parts[1].trim();
boolean isSnapshot = Boolean.parseBoolean(parts[2].trim());
if (id.length() > 1 && url.length() > 1) {
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(isSnapshot, null);
additionalRepos.add(rr);
}
}
}
}
}
/**
* fetch all artifacts
* @return
* @throws MalformedURLException
* @throws ArtifactResolutionException
* @throws DependencyResolutionException
*/
public List<File> fetch() throws MalformedURLException,
DependencyResolutionException, ArtifactResolutionException {
for (Dependency dep : dependencies) {
if (!dep.isLocalFsArtifact()) {
List<ArtifactResult> artifacts = fetchArtifactWithDep(dep);
for (ArtifactResult artifact : artifacts) {
if (dep.isDist()) {
filesDist.add(artifact.getArtifact().getFile());
}
files.add(artifact.getArtifact().getFile());
}
} else {
if (dep.isDist()) {
filesDist.add(new File(dep.getGroupArtifactVersion()));
}
files.add(new File(dep.getGroupArtifactVersion()));
}
}
return files;
}
private List<ArtifactResult> fetchArtifactWithDep(Dependency dep)
throws DependencyResolutionException, ArtifactResolutionException {
Artifact artifact = new DefaultArtifact(
SparkDependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion()));
DependencyFilter classpathFlter = DependencyFilterUtils
.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter(
SparkDependencyResolver.inferScalaVersion(dep.getExclusions()));
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact,
JavaScopes.COMPILE));
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (RemoteRepository repo : additionalRepos) {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {
rr.setAuthentication(auth);
}
collectRequest.addRepository(rr);
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}
public List<File> getFiles() {
return files;
}
public List<File> getFilesDist() {
return filesDist;
}
}

52
spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java → spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java
View file

@ -30,10 +30,9 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.repl.SparkIMain;
import org.apache.zeppelin.dep.AbstractDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
@ -56,21 +55,13 @@ import scala.tools.nsc.util.MergedClassPath;
/**
* Deps resolver.
* Add new dependencies from mvn repo (at runetime) to Zeppelin.
* Add new dependencies from mvn repo (at runtime) to Spark interpreter group.
*/
public class DependencyResolver {
Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
public class SparkDependencyResolver extends AbstractDependencyResolver {
Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class);
private Global global;
private SparkIMain intp;
private SparkContext sc;
private RepositorySystem system = Booter.newRepositorySystem();
private List<RemoteRepository> repos = new LinkedList<RemoteRepository>();
private RepositorySystemSession session;
private DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter(
JavaScopes.COMPILE,
JavaScopes.PROVIDED,
JavaScopes.RUNTIME,
JavaScopes.SYSTEM);
private final String[] exclusions = new String[] {"org.scala-lang:scala-library",
"org.scala-lang:scala-compiler",
@ -80,40 +71,15 @@ public class DependencyResolver {
"org.apache.zeppelin:zeppelin-spark",
"org.apache.zeppelin:zeppelin-server"};
public DependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath,
public SparkDependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath,
String additionalRemoteRepository) {
super(localRepoPath);
this.intp = intp;
this.global = intp.global();
this.sc = sc;
session = Booter.newRepositorySystemSession(system, localRepoPath);
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
addRepoFromProperty(additionalRemoteRepository);
}
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(snapshot, null);
repos.add(rr);
}
}
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
if (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();
return repo;
}
}
}
return null;
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
@ -309,16 +275,16 @@ public class DependencyResolver {
}
/**
*
* @param dependency
* @param excludes list of pattern can either be of the form groupId:artifactId
* @return
* @throws Exception
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception {
Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency));
DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter( JavaScopes.COMPILE );
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
new PatternExclusionsDependencyFilter(inferScalaVersion(excludes));
@ -331,7 +297,7 @@ public class DependencyResolver {
}
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter));
DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}

7
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -36,10 +36,7 @@ class Logger(object):
self.out = ""
def write(self, message):
self.out = self.out + message
def get(self):
return self.out
intp.appendOutput(message)
def reset(self):
self.out = ""
@ -224,7 +221,7 @@ while True :
sc.setJobGroup(jobGroup, "Zeppelin")
eval(compiledCode)
intp.setStatementsFinished(output.get(), False)
intp.setStatementsFinished("", False)
except Py4JJavaError:
excInnerError = traceback.format_exc() # format_tb() does not return the inner exception
innerErrorStart = excInnerError.find("Py4JJavaError:")

2
spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java
View file

@ -60,7 +60,7 @@ public class DepInterpreterTest {
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), null);
}
@After

28
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -28,22 +28,22 @@ import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class SparkInterpreterTest {
public static SparkInterpreter repl;
private InterpreterContext context;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterTest.class);
/**
@ -76,9 +76,21 @@ public class SparkInterpreterTest {
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
@After
@ -177,7 +189,7 @@ public class SparkInterpreterTest {
for (Object oKey : intpProperty.keySet()) {
String key = (String) oKey;
String value = (String) intpProperty.get(key);
repl.logger.debug(String.format("[%s]: [%s]", key, value));
LOGGER.debug(String.format("[%s]: [%s]", key, value));
if (key.startsWith("spark.") && value.isEmpty()) {
assertTrue(String.format("configuration starting from 'spark.' should not be empty. [%s]", key), !sparkConf.contains(key) || !sparkConf.get(key).isEmpty());
}

22
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -25,14 +25,13 @@ import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SparkSqlInterpreterTest {
@ -41,6 +40,8 @@ public class SparkSqlInterpreterTest {
private InterpreterContext context;
private InterpreterGroup intpGroup;
Logger LOGGER = LoggerFactory.getLogger(SparkSqlInterpreterTest.class);
@Before
public void setUp() throws Exception {
Properties p = new Properties();
@ -65,7 +66,17 @@ public class SparkSqlInterpreterTest {
}
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
@After
@ -96,6 +107,7 @@ public class SparkSqlInterpreterTest {
fail("Exception not catched");
} catch (Exception e) {
// okay
LOGGER.info("Exception in SparkSqlInterpreterTest while test ", e);
}
assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from test", context).code());
}

22
spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java → spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java
View file

@ -19,10 +19,10 @@ package org.apache.zeppelin.spark.dep;
import static org.junit.Assert.assertEquals;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.junit.Test;
public class DependencyResolverTest {
public class SparkDependencyResolverTest {
@Test
public void testInferScalaVersion() {
@ -30,23 +30,23 @@ public class DependencyResolverTest {
String scalaVersion = version[0] + "." + version[1];
assertEquals("groupId:artifactId:version",
DependencyResolver.inferScalaVersion("groupId:artifactId:version"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version"));
assertEquals("groupId:artifactId_" + scalaVersion + ":version",
DependencyResolver.inferScalaVersion("groupId::artifactId:version"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId:version"));
assertEquals("groupId:artifactId:version::test",
DependencyResolver.inferScalaVersion("groupId:artifactId:version::test"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version::test"));
assertEquals("*",
DependencyResolver.inferScalaVersion("*"));
SparkDependencyResolver.inferScalaVersion("*"));
assertEquals("groupId:*",
DependencyResolver.inferScalaVersion("groupId:*"));
SparkDependencyResolver.inferScalaVersion("groupId:*"));
assertEquals("groupId:artifactId*",
DependencyResolver.inferScalaVersion("groupId:artifactId*"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId*"));
assertEquals("groupId:artifactId_" + scalaVersion,
DependencyResolver.inferScalaVersion("groupId::artifactId"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId"));
assertEquals("groupId:artifactId_" + scalaVersion + "*",
DependencyResolver.inferScalaVersion("groupId::artifactId*"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId*"));
assertEquals("groupId:artifactId_" + scalaVersion + ":*",
DependencyResolver.inferScalaVersion("groupId::artifactId:*"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId:*"));
}
}

14
tajo/src/test/java/org/apache/zeppelin/tajo/TesterResultSet.java
View file

@ -17,13 +17,16 @@
*/
package org.apache.zeppelin.tajo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InputStream;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.sql.*;
import java.util.Calendar;
import java.util.Map;
import java.io.InputStream;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
/**
* This is borrowed from Apache Commons DBCP2.
@ -31,6 +34,9 @@ import java.io.UnsupportedEncodingException;
* A dummy {@link java.sql.ResultSet}, for testing purposes.
*/
public class TesterResultSet implements ResultSet {
Logger LOGGER = LoggerFactory.getLogger(TesterResultSet.class);
public TesterResultSet(Statement stmt) {
_statement = stmt;
}
@ -262,6 +268,8 @@ public class TesterResultSet implements ResultSet {
return columnName.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
// Impossible. JVMs are required to support UTF-8
LOGGER.error("Exception in TesterResultSet while getBytes, Impossible. JVMs are required to" +
" support UTF-8", e);
return null;
}
}

2
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -91,6 +91,8 @@ The following components are provided under Apache License.
(Apache 2.0) Lucene Suggest (org.apache.lucene:lucene-suggest:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-suggest)
(Apache 2.0) Elasticsearch: Core (org.elasticsearch:elasticsearch:2.1.0 - http://nexus.sonatype.org/oss-repository-hosting.html/parent/elasticsearch)
(Apache 2.0) Joda convert (org.joda:joda-convert:1.2 - http://joda-convert.sourceforge.net)
(Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org)
(Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org)
(Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org)

117
zeppelin-interpreter/pom.xml
View file

@ -97,6 +97,123 @@
<artifactId>commons-lang3</artifactId>
<version>${commons-lang.version}</version>
</dependency>
<!-- Aether :: maven dependency resolution -->
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.0</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.sisu</groupId>
<artifactId>sisu-inject-plexus</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-impl</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-aether-provider</artifactId>
<version>3.0.3</version>
<exclusions>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-spi</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-file</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-wagon</artifactId>
<version>1.12</version>
<exclusions>
<exclusion>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
<version>1.0</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-lightweight</artifactId>
<version>1.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-shared</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http</artifactId>
<version>1.0</version>
<exclusions>
</exclusions>
</dependency>
</dependencies>
<build>

70
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java Normal file
View file

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.resolution.ArtifactResult;
/**
* Abstract dependency resolver.
* Add new dependencies from mvn repo (at runtime) Zeppelin.
*/
public abstract class AbstractDependencyResolver {
protected RepositorySystem system = Booter.newRepositorySystem();
protected List<RemoteRepository> repos = new LinkedList<RemoteRepository>();
protected RepositorySystemSession session;
public AbstractDependencyResolver(String localRepoPath) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
}
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(snapshot, null);
repos.add(rr);
}
}
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
if (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();
return repo;
}
}
}
return null;
}
public abstract List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception;
}

4
spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.File;
@ -64,7 +64,7 @@ public class Booter {
public static RemoteRepository newCentralRepository() {
return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/");
}
public static RemoteRepository newLocalRepository() {
return new RemoteRepository("local",
"default", "file://" + System.getProperty("user.home") + "/.m2/repository");

2
spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Dependency.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.util.LinkedList;
import java.util.List;

41
spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
View file

@ -15,14 +15,12 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.File;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
@ -30,7 +28,6 @@ import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.resolution.ArtifactResolutionException;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
@ -54,11 +51,9 @@ public class DependencyContext {
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
private RemoteRepository mavenLocal = Booter.newLocalRepository();
private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>();
public DependencyContext(String localRepoPath, String additionalRemoteRepository) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
addRepoFromProperty(additionalRemoteRepository);
public DependencyContext(String localRepoPath) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
}
public Dependency load(String lib) {
@ -85,24 +80,6 @@ public class DependencyContext {
filesDist = new LinkedList<File>();
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
for (String repo : repos) {
String[] parts = repo.split(",");
if (parts.length == 3) {
String id = parts[0].trim();
String url = parts[1].trim();
boolean isSnapshot = Boolean.parseBoolean(parts[2].trim());
if (id.length() > 1 && url.length() > 1) {
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(isSnapshot, null);
additionalRepos.add(rr);
}
}
}
}
}
/**
* fetch all artifacts
@ -136,13 +113,12 @@ public class DependencyContext {
private List<ArtifactResult> fetchArtifactWithDep(Dependency dep)
throws DependencyResolutionException, ArtifactResolutionException {
Artifact artifact = new DefaultArtifact(
DependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion()));
Artifact artifact = new DefaultArtifact(dep.getGroupArtifactVersion());
DependencyFilter classpathFlter = DependencyFilterUtils
.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter(
DependencyResolver.inferScalaVersion(dep.getExclusions()));
dep.getExclusions());
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact,
@ -150,16 +126,9 @@ public class DependencyContext {
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (RemoteRepository repo : additionalRepos) {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {
rr.setAuthentication(auth);
}
collectRequest.addRepository(rr);
}

163
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java Normal file
View file

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import java.io.File;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
import org.sonatype.aether.util.artifact.DefaultArtifact;
import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
/**
* Deps resolver.
* Add new dependencies from mvn repo (at runtime) to Zeppelin.
*/
public class DependencyResolver extends AbstractDependencyResolver {
Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
private final String[] exclusions = new String[] {"org.apache.zeppelin:zeppelin-zengine",
"org.apache.zeppelin:zeppelin-interpreter",
"org.apache.zeppelin:zeppelin-server"};
public DependencyResolver(String localRepoPath) {
super(localRepoPath);
}
public List<File> load(String artifact) throws Exception {
return load(artifact, new LinkedList<String>());
}
public List<File> load(String artifact, String destPath) throws Exception {
return load(artifact, new LinkedList<String>(), destPath);
}
public synchronized List<File> load(String artifact, Collection<String> excludes)
throws Exception {
if (StringUtils.isBlank(artifact)) {
// Should throw here
throw new RuntimeException("Invalid artifact to load");
}
// <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>
int numSplits = artifact.split(":").length;
if (numSplits >= 3 && numSplits <= 6) {
return loadFromMvn(artifact, excludes);
} else {
LinkedList<File> libs = new LinkedList<File>();
libs.add(new File(artifact));
return libs;
}
}
public List<File> load(String artifact, Collection<String> excludes, String destPath)
throws Exception {
List<File> libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile: libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}
return libs;
}
private List<File> loadFromMvn(String artifact, Collection<String> excludes) throws Exception {
Collection<String> allExclusions = new LinkedList<String>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
List<ArtifactResult> listOfArtifact;
listOfArtifact = getArtifactsWithDep(artifact, allExclusions);
Iterator<ArtifactResult> it = listOfArtifact.iterator();
while (it.hasNext()) {
Artifact a = it.next().getArtifact();
String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion();
for (String exclude : allExclusions) {
if (gav.startsWith(exclude)) {
it.remove();
break;
}
}
}
List<File> files = new LinkedList<File>();
for (ArtifactResult artifactResult : listOfArtifact) {
files.add(artifactResult.getArtifact().getFile());
logger.info("load {}", artifactResult.getArtifact().getFile().getAbsolutePath());
}
return files;
}
/**
* @param dependency
* @param excludes list of pattern can either be of the form groupId:artifactId
* @return
* @throws Exception
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
new PatternExclusionsDependencyFilter(excludes);
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE));
synchronized (repos) {
for (RemoteRepository repo : repos) {
collectRequest.addRepository(repo);
}
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}
}

4
spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.sonatype.aether.repository.Authentication;
/**
*
@ -70,7 +70,7 @@ public class Repository {
return this;
}
protected Authentication getAuthentication() {
public Authentication getAuthentication() {
Authentication auth = null;
if (this.username != null && this.password != null) {
auth = new Authentication(this.username, this.password);

2
spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/RepositoryListener.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/RepositorySystemFactory.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.apache.maven.repository.internal.DefaultServiceLocator;
import org.apache.maven.wagon.Wagon;

2
spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.PrintStream;
import java.text.DecimalFormat;

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -131,7 +131,7 @@ public abstract class Interpreter {
static Logger logger = LoggerFactory.getLogger(Interpreter.class);
public static Logger logger = LoggerFactory.getLogger(Interpreter.class);
private InterpreterGroup interpreterGroup;
private URL [] classloaderUrls;
protected Properties property;

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.display.GUI;
public class InterpreterContext {
private static final ThreadLocal<InterpreterContext> threadIC =
new ThreadLocal<InterpreterContext>();
public final InterpreterOutput out;
public static InterpreterContext get() {
return threadIC.get();
@ -58,7 +59,8 @@ public class InterpreterContext {
Map<String, Object> config,
GUI gui,
AngularObjectRegistry angularObjectRegistry,
List<InterpreterContextRunner> runners
List<InterpreterContextRunner> runners,
InterpreterOutput out
) {
this.noteId = noteId;
this.paragraphId = paragraphId;
@ -68,6 +70,7 @@ public class InterpreterContext {
this.gui = gui;
this.angularObjectRegistry = angularObjectRegistry;
this.runners = runners;
this.out = out;
}

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -33,6 +33,8 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
public class InterpreterGroup extends LinkedList<Interpreter>{
String id;
Logger LOGGER = Logger.getLogger(InterpreterGroup.class);
AngularObjectRegistry angularObjectRegistry;
RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
@ -100,8 +102,7 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
try {
t.join();
} catch (InterruptedException e) {
Logger logger = Logger.getLogger(InterpreterGroup.class);
logger.error("Can't close interpreter", e);
LOGGER.error("Can't close interpreter", e);
}
}
}
@ -124,8 +125,7 @@ public class InterpreterGroup extends LinkedList<Interpreter>{
try {
t.join();
} catch (InterruptedException e) {
Logger logger = Logger.getLogger(InterpreterGroup.class);
logger.error("Can't close interpreter", e);
LOGGER.error("Can't close interpreter", e);
}
}

249
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java Normal file
View file

@ -0,0 +1,249 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
/**
* InterpreterOutput is OutputStream that supposed to print content on notebook
* in addition to InterpreterResult which used to return from Interpreter.interpret().
*/
public class InterpreterOutput extends OutputStream {
Logger logger = LoggerFactory.getLogger(InterpreterOutput.class);
private final int NEW_LINE_CHAR = '\n';
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
private final List<Object> outList = new LinkedList<Object>();
private InterpreterOutputChangeWatcher watcher;
private final InterpreterOutputListener flushListener;
private InterpreterResult.Type type = InterpreterResult.Type.TEXT;
private boolean firstWrite = true;
public InterpreterOutput(InterpreterOutputListener flushListener) {
this.flushListener = flushListener;
clear();
}
public InterpreterOutput(InterpreterOutputListener flushListener,
InterpreterOutputChangeListener listener) throws IOException {
this.flushListener = flushListener;
clear();
watcher = new InterpreterOutputChangeWatcher(listener);
watcher.start();
}
public InterpreterResult.Type getType() {
return type;
}
public void setType(InterpreterResult.Type type) {
if (this.type != type) {
clear();
flushListener.onUpdate(this, new byte[]{});
this.type = type;
}
}
public void clear() {
synchronized (outList) {
type = InterpreterResult.Type.TEXT;
buffer.reset();
outList.clear();
if (watcher != null) {
watcher.clear();
}
}
}
@Override
public void write(int b) throws IOException {
synchronized (outList) {
buffer.write(b);
if (b == NEW_LINE_CHAR) {
// first time use of this outputstream.
if (firstWrite) {
// clear the output on gui
flushListener.onUpdate(this, new byte[]{});
firstWrite = false;
}
flush();
}
}
}
private byte [] detectTypeFromLine(byte [] byteArray) {
// check output type directive
String line = new String(byteArray);
for (InterpreterResult.Type t : InterpreterResult.Type.values()) {
String typeString = '%' + t.name().toLowerCase();
if ((typeString + "\n").equals(line)) {
setType(t);
byteArray = null;
break;
} else if (line.startsWith(typeString + " ")) {
setType(t);
byteArray = line.substring(typeString.length() + 1).getBytes();
break;
}
}
return byteArray;
}
@Override
public void write(byte [] b) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte [] b, int off, int len) throws IOException {
synchronized (outList) {
for (int i = off; i < len; i++) {
write(b[i]);
}
}
}
/**
* In dev mode, it monitors file and update ZeppelinServer
* @param file
* @throws IOException
*/
public void write(File file) throws IOException {
outList.add(file);
if (watcher != null) {
watcher.watch(file);
}
}
public void write(String string) throws IOException {
write(string.getBytes());
}
/**
* write contents in the resource file in the classpath
* @param url
* @throws IOException
*/
public void write(URL url) throws IOException {
if ("file".equals(url.getProtocol())) {
write(new File(url.getPath()));
} else {
outList.add(url);
}
}
public void writeResource(String resourceName) throws IOException {
// search file under resource dir first for dev mode
File mainResource = new File("./src/main/resources/" + resourceName);
File testResource = new File("./src/test/resources/" + resourceName);
if (mainResource.isFile()) {
write(mainResource);
} else if (testResource.isFile()) {
write(testResource);
} else {
// search from classpath
ClassLoader cl = Thread.currentThread().getContextClassLoader();
if (cl == null) {
cl = this.getClass().getClassLoader();
}
if (cl == null) {
cl = ClassLoader.getSystemClassLoader();
}
write(cl.getResource(resourceName));
}
}
public byte[] toByteArray() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
List<Object> all = new LinkedList<Object>();
synchronized (outList) {
all.addAll(outList);
}
for (Object o : all) {
if (o instanceof File) {
File f = (File) o;
FileInputStream fin = new FileInputStream(f);
copyStream(fin, out);
fin.close();
} else if (o instanceof byte[]) {
out.write((byte[]) o);
} else if (o instanceof Integer) {
out.write((int) o);
} else if (o instanceof URL) {
InputStream fin = ((URL) o).openStream();
copyStream(fin, out);
fin.close();
} else {
// can not handle the object
}
}
out.close();
return out.toByteArray();
}
public void flush() throws IOException {
synchronized (outList) {
buffer.flush();
byte[] bytes = buffer.toByteArray();
bytes = detectTypeFromLine(bytes);
if (bytes != null) {
outList.add(bytes);
if (type == InterpreterResult.Type.TEXT) {
flushListener.onAppend(this, bytes);
}
}
buffer.reset();
}
}
private void copyStream(InputStream in, OutputStream out) throws IOException {
int bufferSize = 8192;
byte[] buffer = new byte[bufferSize];
while (true) {
int bytesRead = in.read(buffer);
if (bytesRead == -1) {
break;
} else {
out.write(buffer, 0, bytesRead);
}
}
}
@Override
public void close() throws IOException {
flush();
if (watcher != null) {
watcher.clear();
watcher.shutdown();
}
}
}

27
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java Normal file
View file

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import java.io.File;
/**
* InterpreterOutputChangeListener
*/
public interface InterpreterOutputChangeListener {
public void fileChanged(File file);
}

140
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcher.java Normal file
View file

@ -0,0 +1,140 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import java.io.File;
import java.io.IOException;
import java.nio.file.ClosedWatchServiceException;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Watch the change for the development mode support
*/
public class InterpreterOutputChangeWatcher extends Thread {
Logger logger = LoggerFactory.getLogger(InterpreterOutputChangeWatcher.class);
private WatchService watcher;
private final List<File> watchFiles = new LinkedList<File>();
private final Map<WatchKey, File> watchKeys = new HashMap<WatchKey, File>();
private InterpreterOutputChangeListener listener;
private boolean stop;
public InterpreterOutputChangeWatcher(InterpreterOutputChangeListener listener)
throws IOException {
watcher = FileSystems.getDefault().newWatchService();
this.listener = listener;
}
public void watch(File file) throws IOException {
String dirString;
if (file.isFile()) {
dirString = file.getParentFile().getAbsolutePath();
} else {
throw new IOException(file.getName() + " is not a file");
}
if (dirString == null) {
dirString = "/";
}
Path dir = FileSystems.getDefault().getPath(dirString);
logger.info("watch " + dir);
WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
synchronized (watchKeys) {
watchKeys.put(key, new File(dirString));
watchFiles.add(file);
}
}
public void clear() {
synchronized (watchKeys) {
for (WatchKey key : watchKeys.keySet()) {
key.cancel();
}
watchKeys.clear();
watchFiles.clear();
}
}
public void shutdown() throws IOException {
stop = true;
clear();
watcher.close();
}
public void run() {
while (!stop) {
WatchKey key = null;
try {
key = watcher.poll(1, TimeUnit.SECONDS);
} catch (InterruptedException | ClosedWatchServiceException e) {
break;
}
if (key == null) {
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind<?> kind = event.kind();
if (kind == OVERFLOW) {
continue;
}
WatchEvent<Path> ev = (WatchEvent<Path>) event;
Path filename = ev.context();
// search for filename
synchronized (watchKeys) {
for (File f : watchFiles) {
if (f.getName().compareTo(filename.toString()) == 0) {
File changedFile;
if (filename.isAbsolute()) {
changedFile = new File(filename.toString());
} else {
changedFile = new File(watchKeys.get(key), filename.toString());
}
logger.info("File change detected " + changedFile.getAbsolutePath());
if (listener != null) {
listener.fileChanged(changedFile);
}
}
}
}
}
boolean valid = key.reset();
if (!valid) {
break;
}
}
}
}

34
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java Normal file
View file

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
/**
* Listen InterpreterOutput buffer flush
*/
public interface InterpreterOutputListener {
/**
* called when newline is detected
* @param line
*/
public void onAppend(InterpreterOutput out, byte[] line);
/**
* when entire output is updated. eg) after detecting new display system
* @param output
*/
public void onUpdate(InterpreterOutput out, byte[] output);
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java
View file

@ -146,4 +146,8 @@ public class InterpreterResult implements Serializable {
this.type = type;
return this;
}
public String toString() {
return "%" + type.name().toLowerCase() + " " + msg;
}
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
View file

@ -101,7 +101,6 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* this method should be used instead of remove()
* @param name
* @param noteId
* @param emit
* @return
*/
public AngularObject removeAndNotifyRemoteProcess(String name, String noteId) {

29
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -48,6 +48,7 @@ import com.google.gson.reflect.TypeToken;
*
*/
public class RemoteInterpreter extends Interpreter {
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
Gson gson = new Gson();
private String interpreterRunner;
@ -60,13 +61,13 @@ public class RemoteInterpreter extends Interpreter {
private int maxPoolSize;
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
int connectTimeout,
int maxPoolSize) {
String className,
String interpreterRunner,
String interpreterPath,
int connectTimeout,
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
@ -74,20 +75,23 @@ public class RemoteInterpreter extends Interpreter {
env = new HashMap<String, String>();
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
Map<String, String> env,
int connectTimeout) {
String className,
String interpreterRunner,
String interpreterPath,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.env = env;
this.connectTimeout = connectTimeout;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
@Override
@ -105,7 +109,8 @@ public class RemoteInterpreter extends Interpreter {
if (intpGroup.getRemoteInterpreterProcess() == null) {
// create new remote process
RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess(
interpreterRunner, interpreterPath, env, connectTimeout);
interpreterRunner, interpreterPath, env, connectTimeout,
remoteInterpreterProcessListener);
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}

27
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -18,29 +18,35 @@
package org.apache.zeppelin.interpreter.remote;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.thrift.TException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/**
*
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
private final RemoteInterpreterProcessListener listener;
private volatile boolean shutdown;
private RemoteInterpreterProcess interpreterProcess;
private InterpreterGroup interpreterGroup;
public RemoteInterpreterEventPoller() {
public RemoteInterpreterEventPoller(RemoteInterpreterProcessListener listener) {
this.listener = listener;
shutdown = false;
}
@ -110,6 +116,24 @@ public class RemoteInterpreterEventPoller extends Thread {
interpreterProcess.getInterpreterContextRunnerPool().run(
runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
// on output append
Map<String, String> outputAppend = gson.fromJson(
event.getData(), new TypeToken<Map<String, String>>() {}.getType());
String noteId = outputAppend.get("noteId");
String paragraphId = outputAppend.get("paragraphId");
String outputToAppend = outputAppend.get("data");
listener.onOutputAppend(noteId, paragraphId, outputToAppend);
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
// on output update
Map<String, String> outputAppend = gson.fromJson(
event.getData(), new TypeToken<Map<String, String>>() {}.getType());
String noteId = outputAppend.get("noteId");
String paragraphId = outputAppend.get("paragraphId");
String outputToUpdate = outputAppend.get("data");
listener.onOutputUpdated(noteId, paragraphId, outputToUpdate);
}
logger.debug("Event from remoteproceess {}", event.getType());
} catch (Exception e) {
@ -124,6 +148,7 @@ public class RemoteInterpreterEventPoller extends Thread {
wait(1000);
}
} catch (InterruptedException ignored) {
logger.info("Error in RemoteInterpreterEventPoller while waitQuietly : ", ignored);
}
}

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -53,10 +53,11 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int connectTimeout;
public RemoteInterpreterProcess(String intpRunner,
String intpDir,
Map<String, String> env,
int connectTimeout) {
this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(), connectTimeout);
String intpDir,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener) {
this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(listener), connectTimeout);
}
RemoteInterpreterProcess(String intpRunner,
@ -121,6 +122,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
"Thread.sleep", e);
}
}
}
@ -177,6 +180,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
client.shutdown();
} catch (Exception e) {
// safely ignore exception while client.shutdown() may terminates remote process
logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
"safely ignore exception while client.shutdown() may terminates remote process", e);
} finally {
if (client != null) {
// no longer used
@ -195,6 +200,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
"Thread.sleep", e);
}
} else {
break;
@ -272,6 +279,8 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
client = getClient();
} catch (NullPointerException e) {
// remote process not started
logger.info("NullPointerException in RemoteInterpreterProcess while " +
"updateRemoteAngularObject getClient, remote process not started", e);
return;
} catch (Exception e) {
logger.error("Can't update angular object", e);

25
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java Normal file
View file

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
/**
* Event from remoteInterpreterProcess
*/
public interface RemoteInterpreterProcessListener {
public void onOutputAppend(String noteId, String paragraphId, String output);
public void onOutputUpdated(String noteId, String paragraphId, String output);
}

77
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -35,15 +35,8 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
@ -114,6 +107,7 @@ public class RemoteInterpreterServer
try {
Thread.sleep(300);
} catch (InterruptedException e) {
logger.info("Exception in RemoteInterpreterServer while shutdown, Thread.sleep", e);
}
}
@ -170,7 +164,7 @@ public class RemoteInterpreterServer
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException
| InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
e.printStackTrace();
logger.error(e.toString(), e);
throw new TException(e);
}
}
@ -225,6 +219,7 @@ public class RemoteInterpreterServer
try {
jobListener.wait(1000);
} catch (InterruptedException e) {
logger.info("Exception in RemoteInterpreterServer while interpret, jobListener.wait", e);
}
}
}
@ -301,7 +296,26 @@ public class RemoteInterpreterServer
try {
InterpreterContext.set(context);
InterpreterResult result = interpreter.interpret(script, context);
return result;
// data from context.out is prepended to InterpreterResult if both defined
String message = "";
context.out.flush();
InterpreterResult.Type outputType = context.out.getType();
byte[] interpreterOutput = context.out.toByteArray();
context.out.clear();
if (interpreterOutput != null && interpreterOutput.length > 0) {
message = new String(interpreterOutput);
}
String interpreterResultMessage = result.message();
if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
message += interpreterResultMessage;
return new InterpreterResult(result.code(), result.type(), message);
} else {
return new InterpreterResult(result.code(), outputType, message);
}
} finally {
InterpreterContext.remove();
}
@ -352,7 +366,8 @@ public class RemoteInterpreterServer
private InterpreterContext convert(RemoteInterpreterContext ric) {
List<InterpreterContextRunner> contextRunners = new LinkedList<InterpreterContextRunner>();
List<InterpreterContextRunner> runners = gson.fromJson(ric.getRunners(),
new TypeToken<List<RemoteInterpreterContextRunner>>(){}.getType());
new TypeToken<List<RemoteInterpreterContextRunner>>() {
}.getType());
for (InterpreterContextRunner r : runners) {
contextRunners.add(new ParagraphRunner(this, r.getNoteId(), r.getParagraphId()));
@ -367,7 +382,40 @@ public class RemoteInterpreterServer
new TypeToken<Map<String, Object>>() {}.getType()),
gson.fromJson(ric.getGui(), GUI.class),
interpreterGroup.getAngularObjectRegistry(),
contextRunners);
contextRunners, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId()));
}
private InterpreterOutput createInterpreterOutput(final String noteId, final String paragraphId) {
return new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", new String(line));
Gson gson = new Gson();
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_APPEND,
gson.toJson(appendOutput)));
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", new String(output));
Gson gson = new Gson();
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_UPDATE,
gson.toJson(appendOutput)));
}
});
}
@ -458,6 +506,7 @@ public class RemoteInterpreterServer
try {
eventQueue.wait(1000);
} catch (InterruptedException e) {
logger.info("Exception in RemoteInterpreterServer while getEvent, eventQueue.wait", e);
}
}
@ -471,7 +520,6 @@ public class RemoteInterpreterServer
/**
* called when object is updated in client (web) side.
* @param className
* @param name
* @param noteId noteId where the update issues
* @param object
@ -502,6 +550,7 @@ public class RemoteInterpreterServer
return;
} catch (Exception e) {
// no luck
logger.info("Exception in RemoteInterpreterServer while angularObjectUpdate, no luck", e);
}
}
@ -513,6 +562,7 @@ public class RemoteInterpreterServer
}.getType());
} catch (Exception e) {
// no lock
logger.info("Exception in RemoteInterpreterServer while angularObjectUpdate, no lock", e);
}
}
@ -547,6 +597,7 @@ public class RemoteInterpreterServer
}.getType());
} catch (Exception e) {
// nolock
logger.info("Exception in RemoteInterpreterServer while angularObjectAdd, nolock", e);
}
// try string object type at last

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtils.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.interpreter.remote;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@ -26,6 +29,7 @@ import java.net.Socket;
*
*/
public class RemoteInterpreterUtils {
static Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreterUtils.class);
public static int findRandomAvailablePortOnAllLocalInterfaces() throws IOException {
int port;
try (ServerSocket socket = new ServerSocket(0);) {
@ -43,6 +47,7 @@ public class RemoteInterpreterUtils {
discover.close();
return true;
} catch (IOException e) {
LOGGER.info("Exception in RemoteInterpreterUtils while checkIfRemoteEndpointAccessible", e);
return false;
}
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteInterpreterContext, RemoteInterpreterContext._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterContext> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext");

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterEvent implements org.apache.thrift.TBase<RemoteInterpreterEvent, RemoteInterpreterEvent._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterEvent> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent");

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java
View file

@ -33,7 +33,9 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum {
ANGULAR_OBJECT_ADD(2),
ANGULAR_OBJECT_UPDATE(3),
ANGULAR_OBJECT_REMOVE(4),
RUN_INTERPRETER_CONTEXT_RUNNER(5);
RUN_INTERPRETER_CONTEXT_RUNNER(5),
OUTPUT_APPEND(6),
OUTPUT_UPDATE(7);
private final int value;
@ -64,6 +66,10 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum {
return ANGULAR_OBJECT_REMOVE;
case 5:
return RUN_INTERPRETER_CONTEXT_RUNNER;
case 6:
return OUTPUT_APPEND;
case 7:
return OUTPUT_UPDATE;
default:
return null;
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterResult implements org.apache.thrift.TBase<RemoteInterpreterResult, RemoteInterpreterResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResult> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult");

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterService {
public interface Iface {

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/FIFOScheduler.java
View file

@ -24,6 +24,8 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* FIFOScheduler runs submitted job sequentially
@ -36,6 +38,8 @@ public class FIFOScheduler implements Scheduler {
Job runningJob = null;
private String name;
static Logger LOGGER = LoggerFactory.getLogger(FIFOScheduler.class);
public FIFOScheduler(String name, ExecutorService executor, SchedulerListener listener) {
this.name = name;
this.executor = executor;
@ -107,6 +111,7 @@ public class FIFOScheduler implements Scheduler {
try {
queue.wait(500);
} catch (InterruptedException e) {
LOGGER.error("Exception in FIFOScheduler while run queue.wait", e);
}
continue;
}

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -78,6 +78,8 @@ public abstract class Job {
Date dateFinished;
Status status;
static Logger LOGGER = LoggerFactory.getLogger(Job.class);
transient boolean aborted = false;
String errorMessage;
@ -173,14 +175,14 @@ public abstract class Job {
dateFinished = new Date();
progressUpdator.terminate();
} catch (NullPointerException e) {
logger().error("Job failed", e);
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
result = e.getMessage();
errorMessage = getStack(e);
dateFinished = new Date();
} catch (Throwable e) {
logger().error("Job failed", e);
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
result = e.getMessage();
@ -249,10 +251,6 @@ public abstract class Job {
return dateFinished;
}
private Logger logger() {
return LoggerFactory.getLogger(Job.class);
}
protected void setResult(Object result) {
this.result = result;
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java
View file

@ -57,6 +57,7 @@ public class JobProgressPoller extends Thread {
try {
Thread.sleep(intervalMs);
} catch (InterruptedException e) {
logger.error("Exception in JobProgressPoller while run Thread.sleep", e);
}
}
}

Some files were not shown because too many files have changed in this diff Show more