Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' of https://github.com/apache/zeppelin

Browse source
This commit is contained in:
Babu Prasad Elumalai 2016-07-26 18:45:04 +00:00
commit aa525530b2
138 changed files with 4518 additions and 2397 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.travis.yml
View file

@ -33,33 +33,41 @@ addons:
matrix:
include:
# Test all modules
# Test all modules with spark-2.0.0-preview and scala 2.11
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''"
env: SCALA_VER="2.11" SPARK_VER="2.0.0-preview" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Dspark.version=2.0.0-preview -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test all modules with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.1
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
@ -68,6 +76,7 @@ before_install:
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
- export R_LIBS='~/R'
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
- ./dev/change_scala_version.sh $SCALA_VER
install:
- mvn $BUILD_FLAG $PROFILE -B
@ -91,6 +100,8 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
- cat spark-*/logs/*
after_script:
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER

37
README.md
View file

@ -69,15 +69,10 @@ First of all, set your proxy configuration on Maven `settings.xml`.
Then, run these commands from shell.
```
export http_proxy=http://localhost:3128
export https_proxy=http://localhost:3128
export HTTP_PROXY=http://localhost:3128
export HTTPS_PROXY=http://localhost:3128
npm config set proxy http://localhost:3128
npm config set https-proxy http://localhost:3128
npm config set registry "http://registry.npmjs.org/"
npm config set strict-ssl false
npm cache clean
git config --global http.proxy http://localhost:3128
git config --global https.proxy http://localhost:3128
git config --global url."http://".insteadOf git://
@ -94,9 +89,7 @@ git config --global --unset url."http://".insteadOf
_Notes:_
- If you are behind NTLM proxy you can use [Cntlm Authentication Proxy](http://cntlm.sourceforge.net/).
- If you are on Windows replace `export` with `set` to set env variables
- Replace `localhost:3128` with the standard pattern `http://user:pwd@host:port`
- For zeppelin-web: currently there is no way to reach Bower main repo through NTLM proxy
- Replace `localhost:3128` with the standard pattern `http://user:pwd@host:port`.
#### Install maven
```
@ -131,6 +124,7 @@ Set spark major version
Available profiles are
```
-Pspark-2.0
-Pspark-1.6
-Pspark-1.5
-Pspark-1.4
@ -164,6 +158,16 @@ Available profiles are
minor version can be adjusted by `-Dhadoop.version=x.x.x`
##### `-Pscala-[version] (optional)`
set scala version (default 2.10)
Available profiles are
```
-Pscala-2.10
-Pscala-2.11
```
##### `-Pyarn` (optional)
enable YARN support for local mode
@ -206,14 +210,17 @@ Available profiles are
Bulid examples under zeppelin-examples directory
#### Example
Here're some examples:
```sh
# basic build
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
# build with spark-2.0, scala-2.11
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
# build with spark-1.6, scala-2.10
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests
@ -294,6 +301,14 @@ And browse [localhost:8080](localhost:8080) in your browser.
For configuration details check __`./conf`__ subdirectory.
### Building for Scala 2.11
To produce a Zeppelin package compiled with Scala 2.11, use the -Pscala-2.11 profile:
```
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Pscala-2.11 -DskipTests clean install
```
### Package
To package the final distribution including the compressed archive, run:

2
SECURITY-README.md
View file

@ -17,7 +17,7 @@ To connect to Zeppelin, users will be asked to enter their credentials. Once log
This a a first step toward full security as implemented by this pull request (https://github.com/apache/zeppelin/pull/53).
# Security setup
1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authcBasic" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections.
1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authc" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections.
2. Secure the Websocket channel : Set to property "zeppelin.anonymous.allowed" to "false" in the file conf/zeppelin-site.xml. You can start by renaming conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
3. Start Zeppelin : bin/zeppelin.sh
4. point your browser to http://localhost:8080

6
cassandra/pom.xml
View file

@ -27,7 +27,7 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-cassandra</artifactId>
<artifactId>zeppelin-cassandra_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Apache Cassandra interpreter</name>
@ -38,14 +38,11 @@
<cassandra.driver.version>3.0.1</cassandra.driver.version>
<snappy.version>1.0.5.4</snappy.version>
<lz4.version>1.3.0</lz4.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<commons-lang.version>3.3.2</commons-lang.version>
<scalate.version>1.7.1</scalate.version>
<cassandra.guava.version>16.0.1</cassandra.guava.version>
<!--TEST-->
<scalatest.version>2.2.4</scalatest.version>
<junit.version>4.12</junit.version>
<achilles.version>3.2.4-Zeppelin</achilles.version>
<assertj.version>1.7.0</assertj.version>
@ -173,6 +170,7 @@
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<id>compile</id>

9
conf/shiro.ini
View file

@ -18,7 +18,7 @@
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password1
admin = password1, admin
user1 = password2, role1, role2
user2 = password3, role3
user3 = password4, role2
@ -58,12 +58,17 @@ shiro.loginUrl = /api/login
role1 = *
role2 = *
role3 = *
admin = *
[urls]
# This section is used for url-based security.
# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide.
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# authc means Form based Auth Security
# To enfore security, comment the line below and uncomment the next one
/api/version = anon
#/api/interpreter/** = authc, roles[admin]
#/api/configurations/** = authc, roles[admin]
#/api/credential/** = authc, roles[admin]
/** = anon
#/** = authc

67
dev/change_scala_version.sh Executable file
View file

@ -0,0 +1,67 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
VALID_VERSIONS=( 2.10 2.11 )
usage() {
echo "Usage: $(basename $0) [-h|--help] <version>
where :
-h| --help Display this help text
valid version values : ${VALID_VERSIONS[*]}
" 1>&2
exit 1
}
if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then
usage
fi
TO_VERSION=$1
check_scala_version() {
for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
echo "Invalid Scala version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2
exit 1
}
check_scala_version "$TO_VERSION"
if [ $TO_VERSION = "2.11" ]; then
FROM_VERSION="2.10"
else
FROM_VERSION="2.11"
fi
sed_i() {
sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
}
export -f sed_i
BASEDIR=$(dirname $0)/..
find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \;
# Also update <scala.binary.version> in parent POM
# Match any scala binary version to ensure idempotency
sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'$TO_VERSION'</' \
"$BASEDIR/pom.xml"

22
dev/publish_release.sh
View file

@ -44,7 +44,7 @@ NC='\033[0m' # No Color
RELEASE_VERSION="$1"
GIT_TAG="$2"
PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
PUBLISH_PROFILES="-Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"
@ -92,13 +92,27 @@ function publish_to_maven() {
tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)"
# build with scala-2.10
echo "mvn clean install -Ppublish-distr \
-Dmaven.repo.local=${tmp_repo} \
-Dmaven.repo.local=${tmp_repo} -Pscala-2.10 \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}"
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" \
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" -Pscala-2.10 \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}
if [[ $? -ne 0 ]]; then
echo "Build failed."
echo "Build with scala 2.10 failed."
exit 1
fi
# build with scala-2.11
"${BASEDIR}/change_scala_version.sh" 2.11
echo "mvn clean install -Ppublish-distr \
-Dmaven.repo.local=${tmp_repo} -Pscala-2.11 \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}"
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" -Pscala-2.11 \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}
if [[ $? -ne 0 ]]; then
echo "Build with scala 2.11 failed."
exit 1
fi

4
docs/Gemfile.lock
View file

@ -27,7 +27,7 @@ GEM
maruku (~> 0.6.0)
pygments.rb (~> 0.5.0)
redcarpet (~> 2.3.0)
safe_yaml (~> 0.9.7)
safe_yaml (~> 1.0.4)
kramdown (1.2.0)
liquid (2.5.4)
listen (1.3.1)
@ -47,7 +47,7 @@ GEM
ffi (>= 0.5.0)
rdiscount (2.1.7)
redcarpet (2.3.0)
safe_yaml (0.9.7)
safe_yaml (1.0.4)
syntax (1.0.0)
yajl-ruby (1.1.0)

8
docs/README.md
View file

@ -1,4 +1,4 @@
## Zeppelin documentation
## Apache Zeppelin documentation
This readme will walk you through building the Zeppelin documentation, which is included here with the Zeppelin source code.
@ -6,15 +6,17 @@ This readme will walk you through building the Zeppelin documentation, which is
## Build documentation
See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
**tl;dr version:**
**Requirements**
```
ruby --version >= 1.9.3
ruby --version >= 2.0.0
gem install bundler
# go to /docs under your Zeppelin source
bundle install
```
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
*On OS X 10.9 you may need to do "xcode-select --install"*

10
docs/development/writingzeppelininterpreter.md
View file

@ -40,15 +40,15 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
## Make your own Interpreter
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should your jars under your interpreter directory with specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpareter-setting.json` in your interpreter jar.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
```
{ZEPPELIN_INTERPRETER_DIR}/{YOUR_OWN_INTERPRETER_DIR}/interpreter-setting.json
```
Here is an example of `interpareter-setting.json` on your own interpreter.
Here is an example of `interpreter-setting.json` on your own interpreter.
```json
[
@ -57,7 +57,7 @@ Here is an example of `interpareter-setting.json` on your own interpreter.
"name": "your-name",
"className": "your.own.interpreter.class",
"properties": {
"propertiies1": {
"properties1": {
"envName": null,
"propertyName": "property.1.name",
"defaultValue": "propertyDefaultValue",
@ -216,4 +216,4 @@ We welcome contribution to a new interpreter. Please follow these few steps:
- Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`).
- Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE).
- Commit your changes and open a [Pull Request](https://github.com/apache/zeppelin/pulls) on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing.

7
docs/install/install.md
View file

@ -93,8 +93,11 @@ mvn clean package -DskipTests [Options]
Here are some examples with several options
```
# basic build
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
# build with spark-2.0, scala-2.11
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
# build with spark-1.6, scala-2.10
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests

17
docs/interpreter/jdbc.md
View file

@ -21,7 +21,8 @@ This interpreter lets you create a JDBC connection to any data source, by now it
* Redshift
* Apache Hive
* Apache Phoenix
* Apache Drill (Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driverde* Apache Tajo
* Apache Drill (Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver))
* Apache Tajo
If someone else used another database please report how it works to improve functionality.
@ -183,7 +184,19 @@ You can modify the interpreter configuration in the `Interpreter` section. The m
<tr>
<td>common.max_result</td>
<td>Max number of SQL result to display to prevent the browser overload. This is common properties for all connections</td>
</tr>
</tr>
<tr>
<td>zeppelin.jdbc.auth.type</td>
<td>Types of authentications' methods supported are SIMPLE, and KERBEROS</td>
</tr>
<tr>
<td>zeppelin.jdbc.principal</td>
<td>The principal name to load from the keytab</td>
</tr>
<tr>
<td>zeppelin.jdbc.keytab.location</td>
<td>The path to the keytab file</td>
</tr>
</table>
To develop this functionality use this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29). For example if a connection needs a schema parameter, it would have to add the property as follows:

35
docs/interpreter/python.md
View file

@ -46,7 +46,7 @@ To access the help, type **help()**
## Python modules
The interpreter can use all modules already installed (with pip, easy_install...)
## Use Zeppelin Dynamic Forms
## Using Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your Python code.
**Zeppelin Dynamic Form can only be used if py4j Python library is installed in your system. If not, you can install it with `pip install py4j`.**
@ -65,6 +65,7 @@ print (z.select("f1",[("o1","1"),("o2","2")],"2"))
print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
```
## Zeppelin features not fully supported by the Python Interpreter
* Interrupt a paragraph execution (`cancel()` method) is currently only supported in Linux and MacOs. If interpreter runs in another operating system (for instance MS Windows) , interrupt a paragraph will close the whole interpreter. A JIRA ticket ([ZEPPELIN-893](https://issues.apache.org/jira/browse/ZEPPELIN-893)) is opened to implement this feature in a next release of the interpreter.
@ -94,7 +95,7 @@ z.show(plt, height='150px')
## Pandas integration
[Zeppelin Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table) provides simple API to visualize data in Pandas DataFrames, same as in Matplotlib.
Apache Zeppelin [Table Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table) provides built-in data visualization capabilities. Python interpreter leverages it to visualize Pandas DataFrames though similar `z.show()` API, same as with [Matplotlib integration](#matplotlib-integration).
Example:
@ -104,6 +105,34 @@ rates = pd.read_csv("bank.csv", sep=";")
z.show(rates)
```
## SQL over Pandas DataFrames
There is a convenience `%python.sql` interpreter that matches Apache Spark experience in Zeppelin and enables usage of SQL language to query [Pandas DataFrames](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html) and visualization of results though built-in [Table Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table).
**Pre-requests**
- Pandas `pip install pandas`
- PandaSQL `pip install -U pandasql`
In case default binded interpreter is Python (first in the interpreter list, under the _Gear Icon_), you can just use it as `%sql` i.e
- first paragraph
```python
import pandas as pd
rates = pd.read_csv("bank.csv", sep=";")
```
- next paragraph
```sql
%sql
SELECT * FROM rates WHERE age < 40
```
Otherwise it can be referred to as `%python.sql`
## Technical description
For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md).
For in-depth technical details on current implementation please refer to [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md).

93
docs/interpreter/shell.md
View file

@ -1,40 +1,55 @@
---
layout: page
title: "Shell Interpreter"
description: "Shell Interpreter"
group: interpreter
---
{% include JB/setup %}
# Shell interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
Shell interpreter uses [Apache Commons Exec](https://commons.apache.org/proper/commons-exec) to execute external processes.
In Zeppelin notebook, you can use ` %sh ` in the beginning of a paragraph to invoke system shell and run commands.
> **Note :** Currently each command runs as the user Zeppelin server is running as.
## Configuration
At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property value for Shell interpreter.
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
<th>Description</th>
</tr>
<tr>
<td>shell.command.timeout.millisecs</td>
<td>60000</td>
<td>Shell command time out in millisecs</td>
</tr>
</table>
## Example
The following example demonstrates the basic usage of Shell in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/shell-example.png" />
---
layout: page
title: "Shell Interpreter"
description: "Shell Interpreter"
group: interpreter
---
{% include JB/setup %}
# Shell interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
Shell interpreter uses [Apache Commons Exec](https://commons.apache.org/proper/commons-exec) to execute external processes.
In Zeppelin notebook, you can use ` %sh ` in the beginning of a paragraph to invoke system shell and run commands.
> **Note :** Currently each command runs as the user Zeppelin server is running as.
## Configuration
At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property value for Shell interpreter.
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
<th>Description</th>
</tr>
<tr>
<td>shell.command.timeout.millisecs</td>
<td>60000</td>
<td>Shell command time out in millisecs</td>
</tr>
<tr>
<td>zeppelin.shell.auth.type</td>
<td></td>
<td>Types of authentications' methods supported are SIMPLE, and KERBEROS</td>
</tr>
<tr>
<td>zeppelin.shell.principal</td>
<td></td>
<td>The principal name to load from the keytab</td>
</tr>
<tr>
<td>zeppelin.shell.keytab.location</td>
<td></td>
<td>The path to the keytab file</td>
</tr>
</table>
## Example
The following example demonstrates the basic usage of Shell in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/shell-example.png" />
If you need further information about **Zeppelin Interpreter Setting** for using Shell interpreter, please read [What is interpreter setting?](../manual/interpreters.html#what-is-interpreter-setting) section first.

35
docs/security/shiroauthentication.md
View file

@ -66,13 +66,17 @@ Finally, you can login using one of the below **username/password** combinations
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
```
admin = password1
user1 = password2
user2 = password3
```
[users]
### 5. Groups and permissions (optional)
In case you want to leverage user groups and permissions, use one of the following configuration for LDAP or AD under `[main]` segment in `shiro.ini`
admin = password1, admin
user1 = password2, role1, role2
user2 = password3, role3
user3 = password4, role2
```
You can set the roles for each users next to the password.
## Groups and permissions (optional)
In case you want to leverage user groups and permissions, use one of the following configuration for LDAP or AD under `[main]` segment in `shiro.ini`.
```
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
@ -101,6 +105,21 @@ finance = *
group1 = *
```
All of above configurations are defined in the `conf/shiro.ini` file.
## Secure your Zeppelin information (optional)
By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
Sometimes you might want to hide these information for your use case.
Since Shiro provides **url-based security**, you can hide the information by commenting or uncommenting these below lines in `conf/shiro.ini`.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).
```
[urls]
/api/interpreter/** = authc, roles[admin]
/api/configurations/** = authc, roles[admin]
/api/credential/** = authc, roles[admin]
```
In this case, only who have `admin` role can see **Interpreter Setting**, **Credential** and **Configuration** information.
If you want to grant this permission to other users, you can change **roles[ ]** as you defined at `[users]` section.
<br/>
> **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file. This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).

21
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java
View file

@ -103,19 +103,6 @@ public class ElasticsearchInterpreter extends Interpreter {
public static final String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster.name";
public static final String ELASTICSEARCH_RESULT_SIZE = "elasticsearch.result.size";
static {
Interpreter.register(
"elasticsearch",
"elasticsearch",
ElasticsearchInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(ELASTICSEARCH_HOST, "localhost", "The host for Elasticsearch")
.add(ELASTICSEARCH_PORT, "9300", "The port for Elasticsearch")
.add(ELASTICSEARCH_CLUSTER_NAME, "elasticsearch", "The cluster name for Elasticsearch")
.add(ELASTICSEARCH_RESULT_SIZE, "10", "The size of the result set of a search query")
.build());
}
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private Client client;
private String host = "localhost";
@ -128,7 +115,13 @@ public class ElasticsearchInterpreter extends Interpreter {
this.host = getProperty(ELASTICSEARCH_HOST);
this.port = Integer.parseInt(getProperty(ELASTICSEARCH_PORT));
this.clusterName = getProperty(ELASTICSEARCH_CLUSTER_NAME);
this.resultSize = Integer.parseInt(getProperty(ELASTICSEARCH_RESULT_SIZE));
try {
this.resultSize = Integer.parseInt(getProperty(ELASTICSEARCH_RESULT_SIZE));
} catch (NumberFormatException e) {
this.resultSize = 10;
logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " +
property.get(ELASTICSEARCH_RESULT_SIZE), e);
}
}
@Override

33
elasticsearch/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,33 @@
[
{
"group": "elasticsearch",
"name": "elasticsearch",
"className": "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter",
"properties": {
"elasticsearch.host": {
"envName": "ELASTICSEARCH_HOST",
"propertyName": "elasticsearch.host",
"defaultValue": "localhost",
"description": "The host for Elasticsearch"
},
"elasticsearch.port": {
"envName": "ELASTICSEARCH_PORT",
"propertyName": "elasticsearch.port",
"defaultValue": "9300",
"description": "The port for Elasticsearch"
},
"elasticsearch.cluster.name": {
"envName": "ELASTICSEARCH_CLUSTER_NAME",
"propertyName": "elasticsearch.cluster.name",
"defaultValue": "elasticsearch",
"description": "The cluster name for Elasticsearch"
},
"elasticsearch.result.size": {
"envName": "ELASTICSEARCH_RESULT_SIZE",
"propertyName": "elasticsearch.result.size",
"defaultValue": "10",
"description": "The size of the result set of a search query"
}
}
}
]

27
file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java
View file

@ -37,17 +37,6 @@ public class HDFSFileInterpreter extends FileInterpreter {
static final String HDFS_USER = "hdfs.user";
static final String HDFS_MAXLENGTH = "hdfs.maxlength";
static {
Interpreter.register(
"hdfs",
"file",
HDFSFileInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(HDFS_URL, "http://localhost:50070/webhdfs/v1/", "The URL for WebHDFS")
.add(HDFS_USER, "hdfs", "The WebHDFS user")
.add(HDFS_MAXLENGTH, "1000", "Maximum number of lines of results fetched").build());
}
Exception exceptionOnConnect = null;
HDFSCommand cmd = null;
Gson gson = null;
@ -262,19 +251,19 @@ public class HDFSFileInterpreter extends FileInterpreter {
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
logger.info("Completion request at position\t" + cursor + " in string " + buf);
final List suggestions = new ArrayList<>();
final List<InterpreterCompletion> suggestions = new ArrayList<>();
if (StringUtils.isEmpty(buf)) {
suggestions.add("ls");
suggestions.add("cd");
suggestions.add("pwd");
suggestions.add(new InterpreterCompletion("ls", "ls"));
suggestions.add(new InterpreterCompletion("cd", "cd"));
suggestions.add(new InterpreterCompletion("pwd", "pwd"));
return suggestions;
}
//part of a command == no spaces
if (buf.split(" ").length == 1){
if ("cd".contains(buf)) suggestions.add("cd");
if ("ls".contains(buf)) suggestions.add("ls");
if ("pwd".contains(buf)) suggestions.add("pwd");
if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd"));
if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls"));
if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd"));
return suggestions;
}
@ -311,7 +300,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1);
//beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length());
suggestions.add(suggestedFinish);
suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish));
}
}
return suggestions;

27
file/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,27 @@
[
{
"group": "file",
"name": "hdfs",
"className": "org.apache.zeppelin.file.HDFSFileInterpreter",
"properties": {
"hdfs.url": {
"envName": null,
"propertyName": "hdfs.url",
"defaultValue": "http://localhost:50070/webhdfs/v1/",
"description": "The URL for WebHDFS"
},
"hdfs.user": {
"envName": null,
"propertyName": "hdfs.user",
"defaultValue": "hdfs",
"description": "The WebHDFS user"
},
"hdfs.maxlength": {
"envName": null,
"propertyName": "hdfs.maxlength",
"defaultValue": "1000",
"description": "Maximum number of lines of results fetched"
}
}
}
]

15
file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java
View file

@ -22,9 +22,13 @@ import com.google.gson.Gson;
import junit.framework.TestCase;
import static org.junit.Assert.*;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.Test;
import org.slf4j.Logger;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import java.lang.Override;
import java.lang.String;
@ -100,6 +104,17 @@ public class HDFSFileInterpreterTest extends TestCase {
// we should be back to first result after all this navigation
assertEquals(result1.message(), result11.message());
// auto completion test
List expectedResultOne = Arrays.asList(
new InterpreterCompletion("ls", "ls"));
List expectedResultTwo = Arrays.asList(
new InterpreterCompletion("pwd", "pwd"));
List<InterpreterCompletion> resultOne = t.completion("l", 0);
List<InterpreterCompletion> resultTwo = t.completion("p", 0);
assertEquals(expectedResultOne, resultOne);
assertEquals(expectedResultTwo, resultTwo);
t.close();
}
}

33
flink/pom.xml
View file

@ -27,7 +27,7 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-flink</artifactId>
<artifactId>zeppelin-flink_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Flink</name>
@ -37,8 +37,6 @@
<properties>
<flink.version>1.0.3</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<flink.scala.binary.version>2.10</flink.scala.binary.version>
<flink.scala.version>2.10.4</flink.scala.version>
<scala.macros.version>2.0.1</scala.macros.version>
</properties>
@ -73,68 +71,71 @@
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${flink.scala.binary.version}</artifactId>
<artifactId>flink-clients_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime_${flink.scala.binary.version}</artifactId>
<artifactId>flink-runtime_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_${flink.scala.binary.version}</artifactId>
<artifactId>flink-scala_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala-shell_${flink.scala.binary.version}</artifactId>
<artifactId>flink-scala-shell_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_${flink.scala.binary.version}</artifactId>
<artifactId>akka-actor_${scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-remote_${flink.scala.binary.version}</artifactId>
<artifactId>akka-remote_${scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-slf4j_${flink.scala.binary.version}</artifactId>
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${flink.scala.binary.version}</artifactId>
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${flink.scala.version}</version>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${flink.scala.version}</version>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${flink.scala.version}</version>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -169,7 +170,7 @@
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.1.4</version>
<version>3.2.2</version>
<executions>
<!-- Run scala compiler in the process-resources phase, so that dependencies on
scala classes can be resolved later in the (Java) compile phase -->
@ -199,7 +200,7 @@
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${flink.scala.version}</artifactId>
<artifactId>paradise_${scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>

30
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -17,6 +17,7 @@
*/
package org.apache.zeppelin.flink;
import java.lang.reflect.InvocationTargetException;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -24,10 +25,7 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import org.apache.flink.api.scala.FlinkILoop;
import org.apache.flink.configuration.Configuration;
@ -45,6 +43,8 @@ import org.slf4j.LoggerFactory;
import scala.Console;
import scala.None;
import scala.Some;
import scala.collection.JavaConversions;
import scala.collection.immutable.Nil;
import scala.runtime.AbstractFunction0;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.IMain;
@ -94,7 +94,7 @@ public class FlinkInterpreter extends Interpreter {
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
binder = (Map<String, Object>) getValue("_binder");
Map<String, Object> binder = (Map<String, Object>) getLastObject();
// import libraries
imain.interpret("import scala.tools.nsc.io._");
@ -103,7 +103,10 @@ public class FlinkInterpreter extends Interpreter {
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
imain.bindValue("env", env);
binder.put("env", env);
imain.interpret("val env = _binder.get(\"env\").asInstanceOf["
+ env.getClass().getName() + "]");
}
private boolean localMode() {
@ -192,16 +195,11 @@ public class FlinkInterpreter extends Interpreter {
return paths;
}
public Object getValue(String name) {
IMain imain = flinkIloop.intp();
Object ret = imain.valueOfTerm(name);
if (ret instanceof None) {
return null;
} else if (ret instanceof Some) {
return ((Some) ret).get();
} else {
return ret;
}
public Object getLastObject() {
Object obj = imain.lastRequest().lineRep().call(
"$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
return obj;
}
@Override

12
ignite/pom.xml
View file

@ -26,16 +26,14 @@
<relativePath>..</relativePath>
</parent>
<artifactId>zeppelin-ignite</artifactId>
<artifactId>zeppelin-ignite_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Apache Ignite interpreter</name>
<url>http://zeppelin.apache.org</url>
<properties>
<ignite.version>1.6.0</ignite.version>
<ignite.scala.binary.version>2.10</ignite.scala.binary.version>
<ignite.scala.version>2.10.4</ignite.scala.version>
<ignite.version>1.5.0.final</ignite.version>
</properties>
<dependencies>
@ -73,19 +71,19 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${ignite.scala.version}</version>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${ignite.scala.version}</version>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${ignite.scala.version}</version>
<version>${scala.version}</version>
</dependency>
<dependency>

18
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -44,6 +44,7 @@ import java.util.Properties;
import scala.Console;
import scala.None;
import scala.Some;
import scala.collection.JavaConversions;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.IMain;
import scala.tools.nsc.interpreter.Results.Result;
@ -174,16 +175,11 @@ public class IgniteInterpreter extends Interpreter {
return paths;
}
public Object getValue(String name) {
Object val = imain.valueOfTerm(name);
if (val instanceof None) {
return null;
} else if (val instanceof Some) {
return ((Some) val).get();
} else {
return val;
}
public Object getLastObject() {
Object obj = imain.lastRequest().lineRep().call(
"$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
return obj;
}
private Ignite getIgnite() {
@ -222,7 +218,7 @@ public class IgniteInterpreter extends Interpreter {
private void initIgnite() {
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getValue("_binder");
Map<String, Object> binder = (Map<String, Object>) getLastObject();
if (getIgnite() != null) {
binder.put("ignite", ignite);

7
jdbc/pom.xml
View file

@ -72,6 +72,13 @@
<version>2.12.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

126
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -17,26 +17,24 @@ package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.jdbc.security.JDBCSecurityImpl;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -169,16 +167,11 @@ public class JDBCInterpreter extends Interpreter {
logger.debug("propertiesMap: {}", propertiesMap);
Connection connection = null;
SqlCompleter sqlCompleter = null;
if (!StringUtils.isAnyEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
JDBCSecurityImpl.createSecureConfiguration(property);
}
for (String propertyKey : propertiesMap.keySet()) {
try {
connection = getConnection(propertyKey);
sqlCompleter = createSqlCompleter(connection);
} catch (Exception e) {
sqlCompleter = createSqlCompleter(null);
}
propertyKeySqlCompleterMap.put(propertyKey, sqlCompleter);
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
}
}
@ -199,7 +192,8 @@ public class JDBCInterpreter extends Interpreter {
return completer;
}
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
public Connection getConnection(String propertyKey, String user)
throws ClassNotFoundException, SQLException, InterpreterException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
@ -215,22 +209,70 @@ public class JDBCInterpreter extends Interpreter {
}
}
if (null == connection) {
Properties properties = propertiesMap.get(propertyKey);
final Properties properties = propertiesMap.get(propertyKey);
logger.info(properties.getProperty(DRIVER_KEY));
Class.forName(properties.getProperty(DRIVER_KEY));
String url = properties.getProperty(URL_KEY);
connection = DriverManager.getConnection(url, properties);
final String url = properties.getProperty(URL_KEY);
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = DriverManager.getConnection(url, properties);
} else {
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = DriverManager.getConnection(url + ";hive.server2.proxy.user=" + user,
properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return DriverManager.getConnection(url, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
}
}
break;
default:
connection = DriverManager.getConnection(url, properties);
}
}
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
return connection;
}
public Statement getStatement(String propertyKey, String paragraphId)
throws SQLException, ClassNotFoundException {
public Statement getStatement(String propertyKey, String paragraphId,
InterpreterContext interpreterContext)
throws SQLException, ClassNotFoundException, InterpreterException {
Connection connection;
if (paragraphIdConnectionMap.containsKey(paragraphId)) {
connection = paragraphIdConnectionMap.get(paragraphId);
if (paragraphIdConnectionMap.containsKey(paragraphId +
interpreterContext.getAuthenticationInfo().getUser())) {
connection = paragraphIdConnectionMap.get(paragraphId +
interpreterContext.getAuthenticationInfo().getUser());
} else {
connection = getConnection(propertyKey);
connection = getConnection(propertyKey, interpreterContext.getAuthenticationInfo().getUser());
}
if (connection == null) {
@ -239,11 +281,13 @@ public class JDBCInterpreter extends Interpreter {
Statement statement = connection.createStatement();
if (isStatementClosed(statement)) {
connection = getConnection(propertyKey);
connection = getConnection(propertyKey, interpreterContext.getAuthenticationInfo().getUser());
statement = connection.createStatement();
}
paragraphIdConnectionMap.put(paragraphId, connection);
paragraphIdStatementMap.put(paragraphId, statement);
paragraphIdConnectionMap.put(paragraphId + interpreterContext.getAuthenticationInfo().getUser(),
connection);
paragraphIdStatementMap.put(paragraphId + interpreterContext.getAuthenticationInfo().getUser(),
statement);
return statement;
}
@ -259,25 +303,35 @@ public class JDBCInterpreter extends Interpreter {
@Override
public void close() {
try {
for (List<Connection> connectionList : propertyKeyUnusedConnectionListMap.values()) {
for (Connection c : connectionList) {
c.close();
try {
c.close();
} catch (Exception e) {
logger.error("Error while closing propertyKeyUnusedConnectionListMap connection...", e);
}
}
}
for (Statement statement : paragraphIdStatementMap.values()) {
statement.close();
try {
statement.close();
} catch (Exception e) {
logger.error("Error while closing paragraphIdStatementMap statement...", e);
}
}
paragraphIdStatementMap.clear();
for (Connection connection : paragraphIdConnectionMap.values()) {
connection.close();
try {
connection.close();
} catch (Exception e) {
logger.error("Error while closing paragraphIdConnectionMap connection...", e);
}
}
paragraphIdConnectionMap.clear();
} catch (SQLException e) {
} catch (Exception e) {
logger.error("Error while closing...", e);
}
}
@ -289,7 +343,7 @@ public class JDBCInterpreter extends Interpreter {
try {
Statement statement = getStatement(propertyKey, paragraphId);
Statement statement = getStatement(propertyKey, paragraphId, interpreterContext);
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
@ -405,7 +459,7 @@ public class JDBCInterpreter extends Interpreter {
String paragraphId = context.getParagraphId();
try {
paragraphIdStatementMap.get(paragraphId).cancel();
paragraphIdStatementMap.get(paragraphId + context.getAuthenticationInfo().getUser()).cancel();
} catch (SQLException e) {
logger.error("Error while cancelling...", e);
}

76
jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java Normal file
View file

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.jdbc.SqlCompleter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Properties;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE;
/**
* Created for org.apache.zeppelin.jdbc.security on 09/07/16.
*/
public class JDBCSecurityImpl {
private static Logger LOGGER = LoggerFactory.getLogger(JDBCSecurityImpl.class);
/***
* @param properties
*/
public static void createSecureConfiguration(Properties properties) {
AuthenticationMethod authType = getAuthtype(properties);
switch (authType) {
case KERBEROS:
Configuration conf = new
org.apache.hadoop.conf.Configuration();
conf.set("hadoop.security.authentication", KERBEROS.toString());
UserGroupInformation.setConfiguration(conf);
try {
UserGroupInformation.loginUserFromKeytab(
properties.getProperty("zeppelin.jdbc.principal"),
properties.getProperty("zeppelin.jdbc.keytab.location")
);
} catch (IOException e) {
LOGGER.error("Failed to get either keytab location or principal name in the " +
"interpreter", e);
}
}
}
public static AuthenticationMethod getAuthtype(Properties properties) {
AuthenticationMethod authType;
try {
authType = AuthenticationMethod.valueOf(properties.getProperty("zeppelin.jdbc.auth.type")
.trim().toUpperCase());
} catch (Exception e) {
LOGGER.error(String.format("Invalid auth.type detected with value %s, defaulting " +
"auth.type to SIMPLE", properties.getProperty("zeppelin.jdbc.auth.type")));
authType = SIMPLE;
}
return authType;
}
}

18
jdbc/src/main/resources/interpreter-setting.json
View file

@ -34,6 +34,12 @@
"defaultValue": "1000",
"description": "Max number of SQL result to display."
},
"zeppelin.jdbc.auth.type": {
"envName": null,
"propertyName": "zeppelin.jdbc.auth.type",
"defaultValue": "",
"description": "If auth type is needed, Example: KERBEROS"
},
"zeppelin.jdbc.concurrent.use": {
"envName": null,
"propertyName": "zeppelin.jdbc.concurrent.use",
@ -46,6 +52,18 @@
"defaultValue": "10",
"description": "Number of concurrent execution"
},
"zeppelin.jdbc.keytab.location": {
"envName": null,
"propertyName": "zeppelin.jdbc.keytab.location",
"defaultValue": "",
"description": "Kerberos keytab location"
},
"zeppelin.jdbc.principal": {
"envName": null,
"propertyName": "zeppelin.jdbc.principal",
"defaultValue": "",
"description": "Kerberos principal"
},
"hive.url": {
"envName": null,
"propertyName": "hive.url",

20
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -40,6 +40,7 @@ import org.apache.zeppelin.jdbc.JDBCInterpreter;
import org.apache.zeppelin.scheduler.FIFOScheduler;
import org.apache.zeppelin.scheduler.ParallelScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
@ -50,6 +51,7 @@ import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
static String jdbcConnection;
InterpreterContext interpreterContext;
private static String getJdbcConnection() throws IOException {
if(null == jdbcConnection) {
@ -84,6 +86,8 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
interpreterContext = new InterpreterContext("", "1", "", "", new AuthenticationInfo(), null, null, null, null,
null, null);
}
@ -126,24 +130,24 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "(fake) select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
// if prefix not found return ERROR and Prefix not found.
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals("Prefix not found.", interpreterResult.message());
}
@Test
public void testDefaultProperties() throws SQLException {
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(getJDBCTestProperties());
assertEquals("org.postgresql.Driver", jdbcInterpreter.getProperty(DEFAULT_DRIVER));
assertEquals("jdbc:postgresql://localhost:5432/", jdbcInterpreter.getProperty(DEFAULT_URL));
assertEquals("gpadmin", jdbcInterpreter.getProperty(DEFAULT_USER));
assertEquals("", jdbcInterpreter.getProperty(DEFAULT_PASSWORD));
assertEquals("1000", jdbcInterpreter.getProperty(COMMON_MAX_LINE));
}
@Test
public void testSelectQuery() throws SQLException, IOException {
Properties properties = new Properties();
@ -158,7 +162,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table WHERE ID in ('a', 'b')";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
@ -179,7 +183,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table WHERE ID = 'c'";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
@ -202,7 +206,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
@ -244,6 +248,8 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
jdbcInterpreter.interpret("", interpreterContext);
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("SEL", 0);
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("SELECT", "SELECT");

0
kylin/pom.xml Executable file → Normal file
View file

31
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
View file

@ -33,6 +33,7 @@ import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import java.nio.charset.Charset;
@ -348,18 +349,26 @@ public class LivyHelper {
RestTemplate restTemplate = getRestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("Content-Type", "application/json");
headers.add("X-Requested-By", "zeppelin");
ResponseEntity<String> response = null;
if (method.equals("POST")) {
HttpEntity<String> entity = new HttpEntity<String>(jsonData, headers);
response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("GET")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("DELETE")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class);
try {
if (method.equals("POST")) {
HttpEntity<String> entity = new HttpEntity<String>(jsonData, headers);
response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("GET")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("DELETE")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class);
}
} catch (HttpClientErrorException e) {
response = new ResponseEntity(e.getResponseBodyAsString(), e.getStatusCode());
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), e.getResponseBodyAsString()));
}
if (response == null) {
return null;

4
markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java
View file

@ -40,10 +40,6 @@ public class Markdown extends Interpreter {
private Markdown4jProcessor md;
static final Logger LOGGER = LoggerFactory.getLogger(Markdown.class);
static {
Interpreter.register("md", Markdown.class.getName());
}
public Markdown(Properties property) {
super(property);
}

8
markdown/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,8 @@
[
{
"group": "md",
"name": "md",
"className": "org.apache.zeppelin.markdown.Markdown",
"properties": null
}
]

1
markdown/src/test/java/org/apache/zeppelin/markdown/MarkdownTest.java
View file

@ -43,7 +43,6 @@ public class MarkdownTest {
md.open();
InterpreterResult result = md.interpret("This is ~~deleted~~ text", null);
assertEquals("<p>This is <s>deleted</s> text</p>\n", result.message());
System.out.println(MarkdownTest.class.getName());
}
}

67
notebook/2BQA35CJZ/note.json
View file

File diff suppressed because one or more lines are too long

64
pom.xml Executable file → Normal file
View file

@ -17,7 +17,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.maven-v4_0_0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
@ -79,11 +79,16 @@
</modules>
<properties>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<scalatest.version>2.2.4</scalatest.version>
<scalacheck.version>1.12.5</scalacheck.version>
<slf4j.version>1.7.10</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<libthrift.version>0.9.2</libthrift.version>
<gson.version>2.2</gson.version>
<guava.version>19.0</guava.version>
<guava.version>15.0</guava.version>
<jetty.version>9.2.15.v20160210</jetty.version>
<PermGen>64m</PermGen>
@ -93,7 +98,6 @@
<dependencyManagement>
<dependencies>
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -136,7 +140,6 @@
<version>2.5</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
@ -155,14 +158,12 @@
<version>1.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
@ -181,7 +182,6 @@
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@ -388,12 +388,25 @@
</executions>
</plugin>
<!--TODO(alex): make part of the build and reconcile conflicts <plugin>
<groupId>com.ning.maven.plugins</groupId> <artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.4</version> <executions> <execution> <id>default</id> <phase>verify</phase>
<goals> <goal>check</goal> </goals> </execution> </executions> <configuration>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict> </configuration>
</plugin> -->
<!--TODO(alex): make part of the build and reconcile conflicts
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
<version>1.0.4</version>
<executions>
<execution>
<id>default</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<failBuildInCaseOfConflict>true</failBuildInCaseOfConflict>
</configuration>
</plugin>
-->
</plugins>
<pluginManagement>
@ -412,7 +425,7 @@
<exclude>.github/*</exclude>
<exclude>.gitignore</exclude>
<exclude>.repository/</exclude>
<exclude>.Rhistory</exclude>
<exclude>.Rhistory</exclude>
<exclude>**/*.diff</exclude>
<exclude>**/*.patch</exclude>
<exclude>**/*.avsc</exclude>
@ -635,6 +648,28 @@
</build>
<profiles>
<profile>
<id>scala-2.10</id>
<activation>
<property><name>!scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
</profile>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<properties>
<scala.version>2.11.7</scala.version>
<scala.binary.version>2.11</scala.binary.version>
</properties>
</profile>
<profile>
<id>vendor-repo</id>
<repositories>
@ -710,7 +745,6 @@
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>

10
python/README.md
View file

@ -6,6 +6,14 @@ Current interpreter implementation spawns new system python process through `Pro
# Details
- **UnitTests**
To run full suit of tests, including ones that depend on real Python interpreter AND external libraries installed (like Pandas, Pandasql, etc) do
```
mvn -Dpython.test.exclude='' test -pl python -am
```
- **Py4j support**
[Py4j](https://www.py4j.org/) enables Python programs to dynamically access Java objects in a JVM.
@ -40,3 +48,5 @@ Current interpreter implementation spawns new system python process through `Pro
* JavaBuilder can't send SIGINT signal to interrupt paragraph execution. Therefore interpreter directly send a `kill SIGINT PID` to python process to interrupt execution. Python process catch SIGINT signal with some code defined in bootstrap.py
* Matplotlib display feature is made with SVG export (in string) and then displays it with html code.
* `%python.sql` support for Pandas DataFrames is optional and provided using https://github.com/yhat/pandasql if user have one installed

5
python/pom.xml
View file

@ -35,7 +35,10 @@
<properties>
<py4j.version>0.9.2</py4j.version>
<python.test.exclude>**/PythonInterpreterWithPythonInstalledTest.java</python.test.exclude>
<python.test.exclude>
**/PythonInterpreterWithPythonInstalledTest.java,
**/PythonInterpreterPandasSqlTest.java
</python.test.exclude>
</properties>
<dependencies>

17
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -204,15 +204,20 @@ public class PythonInterpreter extends Interpreter {
}
private String sendCommandToPython(String cmd) {
/**
* Sends given text to Python interpreter, blocks and returns the output
* @param cmd Python expression text
* @return output
*/
String sendCommandToPython(String cmd) {
String output = "";
LOG.info("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 200) + "..." : cmd));
LOG.debug("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 200) + "..." : cmd));
try {
output = process.sendAndGetResult(cmd);
} catch (IOException e) {
LOG.error("Error when sending commands to python process", e);
}
//logger.info("Got : \n" + output);
LOG.debug("Got : \n" + output);
return output;
}
@ -243,11 +248,7 @@ public class PythonInterpreter extends Interpreter {
public Boolean isPy4jInstalled() {
String output = sendCommandToPython("\n\nimport py4j\n");
if (output.contains("ImportError")) {
return false;
} else {
return true;
}
return !output.contains("ImportError");
}
private int findRandomOpenPortOnAllLocalInterfaces() {

115
python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java Normal file
View file

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import java.io.IOException;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SQL over Pandas DataFrame interpreter for %python group
*
* Match experience of %sparpk.sql over Spark DataFrame
*/
public class PythonInterpreterPandasSql extends Interpreter {
private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreterPandasSql.class);
private String SQL_BOOTSTRAP_FILE_PY = "/bootstrap_sql.py";
public PythonInterpreterPandasSql(Properties property) {
super(property);
}
PythonInterpreter getPythonInterpreter() {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
python = (PythonInterpreter) p;
if (lazy != null) {
lazy.open();
}
return python;
}
@Override
public void open() {
LOG.info("Open Python SQL interpreter instance: {}", this.toString());
try {
LOG.info("Bootstrap {} interpreter with {}", this.toString(), SQL_BOOTSTRAP_FILE_PY);
PythonInterpreter python = getPythonInterpreter();
python.bootStrapInterpreter(SQL_BOOTSTRAP_FILE_PY);
} catch (IOException e) {
LOG.error("Can't execute " + SQL_BOOTSTRAP_FILE_PY + " to import SQL dependencies", e);
}
}
/**
* Checks if Python dependencies pandas and pandasql are installed
* @return True if they are
*/
boolean isPandasAndPandasqlInstalled() {
PythonInterpreter python = getPythonInterpreter();
String output = python.sendCommandToPython("\n\nimport pandas\nimport pandasql\n");
return !output.contains("ImportError");
}
@Override
public void close() {
LOG.info("Close Python SQL interpreter instance: {}", this.toString());
Interpreter python = getPythonInterpreter();
python.close();
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
Interpreter python = getPythonInterpreter();
return python.interpret("z.show(pysqldf('" + st + "'))", context);
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

25
python/src/main/resources/bootstrap.py
View file

@ -72,6 +72,8 @@ plt.close()
print ('''<pre>z.show(plt,width='50px')
z.show(plt,height='150px') </pre></div>''')
print ('<h3>Pandas DataFrame</h3>')
print ('<div> You need to have Pandas module installed ')
print ('to use this functionality (pip install pandas) !</div><br/>')
print """
<div>The interpreter can visualize Pandas DataFrame
with the function z.show()
@ -81,6 +83,27 @@ df = pd.read_csv("bank.csv", sep=";")
z.show(df)
</pre></div>
"""
print ('<h3>SQL over Pandas DataFrame</h3>')
print ('<div> You need to have Pandas&Pandasql modules installed ')
print ('to use this functionality (pip install pandas pandasql) !</div><br/>')
print """
<div>Python interpreter group includes %sql interpreter that can query
Pandas DataFrames using SQL and visualize results using Zeppelin Table Display System
<pre>
%python
import pandas as pd
df = pd.read_csv("bank.csv", sep=";")
</pre>
<br />
<pre>
%python.sql
%sql
SELECT * from df LIMIT 5
</pre></div>
"""
class PyZeppelinContext(object):
""" If py4j is detected, these class will be override
@ -109,6 +132,8 @@ class PyZeppelinContext(object):
# `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
# and so a dependency on pandas
self.show_dataframe(p, **kwargs)
elif hasattr(p, '__call__'):
p() #error reporting
def show_dataframe(self, df, **kwargs):
"""Pretty prints DF using Table Display System

28
python/src/main/resources/bootstrap_sql.py Normal file
View file

@ -0,0 +1,28 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Setup SQL over Pandas DataFrames
# It requires next dependencies to be installed:
# - pandas
# - pandasql
from __future__ import print_function
try:
from pandasql import sqldf
pysqldf = lambda q: sqldf(q, globals())
except ImportError:
pysqldf = lambda q: print("Can not run SQL over Pandas DataFrame" +
"Make sure 'pandas' and 'pandasql' libraries are installed")

6
python/src/main/resources/interpreter-setting.json
View file

@ -17,5 +17,11 @@
"description": "Max number of dataframe rows to display."
}
}
},
{
"group": "python",
"name": "sql",
"className": "org.apache.zeppelin.python.PythonPandasSqlInterpreter",
"properties": { }
}
]

156
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java Normal file
View file

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
/**
* In order for this test to work, test env must have installed:
* <ol>
* - <li>Python</li>
* - <li>NumPy</li>
* - <li>Pandas</li>
* - <li>PandaSql</li>
* <ol>
*
* To run manually on such environment, use:
* <code>
* mvn -Dpython.test.exclude='' test -pl python -am
* </code>
*/
public class PythonInterpreterPandasSqlTest {
private InterpreterGroup intpGroup;
private PythonInterpreterPandasSql sql;
private PythonInterpreter python;
private InterpreterContext context;
@Before
public void setUp() throws Exception {
Properties p = new Properties();
p.setProperty("zeppelin.python", "python");
p.setProperty("zeppelin.python.maxResult", "100");
intpGroup = new InterpreterGroup();
python = new PythonInterpreter(p);
python.setInterpreterGroup(intpGroup);
python.open();
sql = new PythonInterpreterPandasSql(p);
sql.setInterpreterGroup(intpGroup);
intpGroup.put("note", Arrays.asList(python, sql));
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(
new InterpreterOutputListener() {
@Override public void onAppend(InterpreterOutput out, byte[] line) {}
@Override public void onUpdate(InterpreterOutput out, byte[] output) {}
}));
//important to be last step
sql.open();
//it depends on python interpreter presence in the same group
}
@Test
public void dependenciesAreInstalled() {
InterpreterResult ret = python.interpret("import pandas\nimport pandasql\nimport numpy\n", context);
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
}
@Test
public void errorMessageIfDependenciesNotInstalled() {
InterpreterResult ret;
// given
ret = python.interpret(
"pysqldf = lambda q: print('Can not execute SQL as Python dependency is not installed')",
context);
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
// when
ret = sql.interpret("SELECT * from something", context);
// then
assertNotNull(ret);
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().contains("dependency is not installed"));
}
@Test
public void sqlOverTestDataPrintsTable() {
InterpreterResult ret;
// given
//String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34";
ret = python.interpret("import pandas as pd", context);
ret = python.interpret("import numpy as np", context);
// DataFrame df2 \w test data
ret = python.interpret("df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), "+
"'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
//when
ret = sql.interpret("select name, age from df2 where age < 40", context);
//then
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(ret.message(), Type.TABLE, ret.type());
//assertEquals(expectedTable, ret.message()); //somehow it's same but not equal
assertTrue(ret.message().indexOf("moon\t33") > 0);
assertTrue(ret.message().indexOf("park\t34") > 0);
assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from df2", context).code());
}
@Test
public void badSqlSyntaxFails() {
//when
InterpreterResult ret = sql.interpret("select wrong syntax", context);
//then
assertNotNull("Interpreter returned 'null'", ret);
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().length() > 0);
}
}

2
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -187,7 +187,7 @@ public class PythonInterpreterTest {
s.connect(sa, 10000);
connected = true;
} catch (IOException e) {
LOG.error("Can't open connection to " + sa, e);
//LOG.warn("Can't open connection to " + sa, e);
}
return connected;
}

2
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
View file

@ -36,7 +36,7 @@ import org.junit.Test;
*
* or
* <code>
* mvn -Dpython.test.exclude='' test -pl python
* mvn -Dpython.test.exclude='' test -pl python -am
* </code>
*/
public class PythonInterpreterWithPythonInstalledTest {

37
r/pom.xml
View file

@ -27,7 +27,7 @@
<relativePath>..</relativePath>
</parent>
<artifactId>zeppelin-zrinterpreter</artifactId>
<artifactId>zeppelin-zrinterpreter_2.10</artifactId>
<packaging>jar</packaging>
<name>Zeppelin: R Interpreter</name>
<description>R Interpreter for Zeppelin</description>
@ -36,8 +36,6 @@
<script.extension>.sh</script.extension>
<path.separator>/</path.separator>
<spark.version>1.4.1</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
<dependencies>
@ -58,7 +56,7 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-spark-dependencies</artifactId>
<artifactId>zeppelin-spark-dependencies_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
@ -70,7 +68,7 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-spark</artifactId>
<artifactId>zeppelin-spark_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
@ -118,13 +116,13 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>2.2.4</version>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
<version>1.12.5</version>
<version>${scalacheck.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@ -376,4 +374,29 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>scala-2.10</id>
<activation>
<property><name>!scala-2.11</name></property>
</activation>
<properties>
<extra.source.dir>src/main/scala-2.10</extra.source.dir>
<extra.testsource.dir>src/test/scala-2.10</extra.testsource.dir>
</properties>
</profile>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<properties>
<extra.source.dir>src/main/scala-2.11</extra.source.dir>
<extra.testsource.dir>src/test/scala/scala-2.11</extra.testsource.dir>
</properties>
</profile>
</profiles>
</project>

18
scalding/pom.xml
View file

@ -27,14 +27,13 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-scalding</artifactId>
<artifactId>zeppelin-scalding_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Scalding interpreter</name>
<url>http://zeppelin.apache.org</url>
<properties>
<scala.version>2.11.8</scala.version>
<hadoop.version>2.6.0</hadoop.version>
<scalding.version>0.16.1-RC1</scalding.version>
</properties>
@ -74,43 +73,43 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-core_2.11</artifactId>
<artifactId>scalding-core_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-args_2.11</artifactId>
<artifactId>scalding-args_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-date_2.11</artifactId>
<artifactId>scalding-date_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-commons_2.11</artifactId>
<artifactId>scalding-commons_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-avro_2.11</artifactId>
<artifactId>scalding-avro_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-parquet_2.11</artifactId>
<artifactId>scalding-parquet_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>scalding-repl_2.11</artifactId>
<artifactId>scalding-repl_${scala.binary.version}</artifactId>
<version>${scalding.version}</version>
</dependency>
@ -199,6 +198,7 @@
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<id>compile</id>

6
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -38,6 +38,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.shell.security.ShellSecurityImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -57,8 +58,11 @@ public class ShellInterpreter extends Interpreter {
@Override
public void open() {
LOGGER.info("Command timeout property: {}", TIMEOUT_PROPERTY);
LOGGER.info("Command timeout property: {}", getProperty(TIMEOUT_PROPERTY));
executors = new HashMap<String, DefaultExecutor>();
if (!StringUtils.isAnyEmpty(getProperty("zeppelin.shell.auth.type"))) {
ShellSecurityImpl.createSecureConfiguration(getProperty(), shell);
}
}
@Override

59
shell/src/main/java/org/apache/zeppelin/shell/security/ShellSecurityImpl.java Normal file
View file

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.shell.security;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
/***
* Shell security helper
*/
public class ShellSecurityImpl {
private static Logger LOGGER = LoggerFactory.getLogger(ShellSecurityImpl.class);
public static void createSecureConfiguration(Properties properties, String shell) {
String authType = properties.getProperty("zeppelin.shell.auth.type")
.trim().toUpperCase();
switch (authType) {
case "KERBEROS":
CommandLine cmdLine = CommandLine.parse(shell);
cmdLine.addArgument("-c", false);
String kinitCommand = String.format("kinit -k -t %s %s",
properties.getProperty("zeppelin.shell.keytab.location"),
properties.getProperty("zeppelin.shell.principal"));
cmdLine.addArgument(kinitCommand, false);
DefaultExecutor executor = new DefaultExecutor();
try {
int exitVal = executor.execute(cmdLine);
} catch (Exception e) {
LOGGER.error("Unable to run kinit for zeppelin user " + kinitCommand, e);
throw new InterpreterException(e);
}
}
}
}

18
shell/src/main/resources/interpreter-setting.json
View file

@ -9,6 +9,24 @@
"propertyName": "shell.command.timeout.millisecs",
"defaultValue": "60000",
"description": "Shell command time out in millisecs. Default = 60000"
},
"zeppelin.shell.auth.type": {
"envName": null,
"propertyName": "zeppelin.shell.auth.type",
"defaultValue": "",
"description": "If auth type is needed, Example: KERBEROS"
},
"zeppelin.shell.keytab.location": {
"envName": null,
"propertyName": "zeppelin.shell.keytab.location",
"defaultValue": "",
"description": "Kerberos keytab location"
},
"zeppelin.shell.principal": {
"envName": null,
"propertyName": "zeppelin.shell.principal",
"defaultValue": "",
"description": "Kerberos principal"
}
}
}

36
spark-dependencies/pom.xml
View file

@ -28,7 +28,7 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-spark-dependencies</artifactId>
<artifactId>zeppelin-spark-dependencies_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Spark dependencies</name>
@ -36,10 +36,16 @@
<url>http://zeppelin.apache.org</url>
<properties>
<spark.version>1.4.1</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<!-- library version defined in this section brought from spark 1.4.1 and it's dependency.
Therefore changing only spark.version is not going to be enough when this module
support new version of spark to make the new version as default supported version.
Each profile (spark-2.0, spark-1.6, etc) will overrides necessary dependency version.
So we'll make one of those profile 'activateByDefault' to make it default supported version
instead of changing spark.version in this section.
-->
<spark.version>1.4.1</spark.version>
<hadoop.version>2.3.0</hadoop.version>
<yarn.version>${hadoop.version}</yarn.version>
<avro.version>1.7.7</avro.version>
@ -285,12 +291,6 @@
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-twitter_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
@ -511,9 +511,6 @@
<profile>
<id>spark-1.6</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>1.6.1</spark.version>
<py4j.version>0.9</py4j.version>
@ -523,6 +520,19 @@
</properties>
</profile>
<profile>
<id>spark-2.0</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.0.0</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.1</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>
<profile>
<id>hadoop-0.23</id>
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a

14
spark/pom.xml
View file

@ -27,7 +27,7 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-spark</artifactId>
<artifactId>zeppelin-spark_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Spark</name>
@ -39,8 +39,6 @@
<mockito.version>1.10.19</mockito.version>
<powermock.version>1.6.4</powermock.version>
<spark.version>1.6.2</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
<dependencies>
@ -54,11 +52,11 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<!-- dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-display</artifactId>
<artifactId>zeppelin-display_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependency -->
<dependency>
<groupId>${project.groupId}</groupId>
@ -250,7 +248,7 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>2.2.4</version>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
@ -412,6 +410,7 @@
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<id>compile</id>
@ -440,7 +439,6 @@
</build>
<profiles>
<!-- to deactivate 'exclude-sparkr' automatically when 'spark' is activated -->
<profile>
<id>sparkr</id>

96
spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java
View file

@ -21,21 +21,22 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.reflect.Type;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.spark.repl.SparkILoop;
import org.apache.spark.repl.SparkIMain;
import org.apache.spark.repl.SparkJLineCompletion;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
@ -51,9 +52,12 @@ import scala.Console;
import scala.None;
import scala.Some;
import scala.collection.convert.WrapAsJava$;
import scala.collection.JavaConversions;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.Completion.Candidates;
import scala.tools.nsc.interpreter.Completion.ScalaCompleter;
import scala.tools.nsc.interpreter.IMain;
import scala.tools.nsc.interpreter.Results;
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
import scala.tools.nsc.settings.MutableSettings.PathSetting;
@ -64,10 +68,17 @@ import scala.tools.nsc.settings.MutableSettings.PathSetting;
*
*/
public class DepInterpreter extends Interpreter {
private SparkIMain intp;
/**
* intp - org.apache.spark.repl.SparkIMain (scala 2.10)
* intp - scala.tools.nsc.interpreter.IMain; (scala 2.11)
*/
private Object intp;
private ByteArrayOutputStream out;
private SparkDependencyContext depc;
private SparkJLineCompletion completor;
/**
* completor - org.apache.spark.repl.SparkJLineCompletion (scala 2.10)
*/
private Object completor;
private SparkILoop interpreter;
static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class);
@ -103,7 +114,7 @@ public class DepInterpreter extends Interpreter {
@Override
public void close() {
if (intp != null) {
intp.close();
Utils.invokeMethod(intp, "close");
}
}
@ -149,31 +160,53 @@ public class DepInterpreter extends Interpreter {
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
interpreter = new SparkILoop(null, new PrintWriter(out));
interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out));
interpreter.settings_$eq(settings);
interpreter.createInterpreter();
intp = interpreter.intp();
intp.setContextClassLoader();
intp.initializeSynchronous();
intp = Utils.invokeMethod(interpreter, "intp");
if (Utils.isScala2_10()) {
Utils.invokeMethod(intp, "setContextClassLoader");
Utils.invokeMethod(intp, "initializeSynchronous");
}
depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
completor = new SparkJLineCompletion(intp);
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getValue("_binder");
if (Utils.isScala2_10()) {
completor = Utils.instantiateClass(
"org.apache.spark.repl.SparkJLineCompletion",
new Class[]{Utils.findClass("org.apache.spark.repl.SparkIMain")},
new Object[]{intp});
}
interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder;
if (Utils.isScala2_10()) {
binder = (Map<String, Object>) getValue("_binder");
} else {
binder = (Map<String, Object>) getLastObject();
}
binder.put("depc", depc);
intp.interpret("@transient val z = "
interpret("@transient val z = "
+ "_binder.get(\"depc\")"
+ ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]");
}
private Results.Result interpret(String line) {
return (Results.Result) Utils.invokeMethod(
intp,
"interpret",
new Class[] {String.class},
new Object[] {line});
}
public Object getValue(String name) {
Object ret = intp.valueOfTerm(name);
Object ret = Utils.invokeMethod(
intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name});
if (ret instanceof None) {
return null;
} else if (ret instanceof Some) {
@ -183,6 +216,13 @@ public class DepInterpreter extends Interpreter {
}
}
public Object getLastObject() {
IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest");
Object obj = r.lineRep().call("$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
return obj;
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
PrintStream printStream = new PrintStream(out);
@ -198,7 +238,7 @@ public class DepInterpreter extends Interpreter {
"restart Zeppelin/Interpreter" );
}
scala.tools.nsc.interpreter.Results.Result ret = intp.interpret(st);
scala.tools.nsc.interpreter.Results.Result ret = interpret(st);
Code code = getResultCode(ret);
try {
@ -245,17 +285,21 @@ public class DepInterpreter extends Interpreter {
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
ScalaCompleter c = completor.completer();
Candidates ret = c.complete(buf, cursor);
if (Utils.isScala2_10()) {
ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completor, "completer");
Candidates ret = c.complete(buf, cursor);
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
}
return completions;
} else {
return new LinkedList<InterpreterCompletion>();
}
return completions;
}
private List<File> currentClassPath() {

11
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -48,8 +48,6 @@ import org.apache.spark.sql.SQLContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
@ -530,6 +528,15 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public Object getSparkSession() {
SparkInterpreter intp = getSparkInterpreter();
if (intp == null) {
return null;
} else {
return intp.getSparkSession();
}
}
public SparkConf getSparkConf() {
JavaSparkContext sc = getJavaSparkContext();
if (sc == null) {

552
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -19,24 +19,25 @@ package org.apache.zeppelin.spark;
import java.io.File;
import java.io.PrintWriter;
import java.lang.reflect.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Joiner;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.spark.HttpServer;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkEnv;
import org.apache.spark.repl.SparkCommandLine;
import org.apache.spark.SecurityManager;
import org.apache.spark.repl.SparkILoop;
import org.apache.spark.repl.SparkIMain;
import org.apache.spark.repl.SparkJLineCompletion;
import org.apache.spark.scheduler.ActiveJob;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.Pool;
@ -45,7 +46,6 @@ import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
@ -72,9 +72,12 @@ import scala.collection.convert.WrapAsScala;
import scala.collection.mutable.HashMap;
import scala.collection.mutable.HashSet;
import scala.reflect.io.AbstractFile;
import scala.tools.nsc.Global;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.Completion.Candidates;
import scala.tools.nsc.interpreter.Completion.ScalaCompleter;
import scala.tools.nsc.interpreter.IMain;
import scala.tools.nsc.interpreter.Results;
import scala.tools.nsc.settings.MutableSettings;
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
import scala.tools.nsc.settings.MutableSettings.PathSetting;
@ -88,10 +91,16 @@ public class SparkInterpreter extends Interpreter {
private ZeppelinContext z;
private SparkILoop interpreter;
private SparkIMain intp;
/**
* intp - org.apache.spark.repl.SparkIMain (scala 2.10)
* intp - scala.tools.nsc.interpreter.IMain; (scala 2.11)
*/
private Object intp;
private SparkConf conf;
private static SparkContext sc;
private static SQLContext sqlc;
private static SparkEnv env;
private static Object sparkSession; // spark 2.x
private static JobProgressListener sparkListener;
private static AbstractFile classOutputDir;
private static Integer sharedInterpreterLock = new Integer(0);
@ -99,10 +108,16 @@ public class SparkInterpreter extends Interpreter {
private SparkOutputStream out;
private SparkDependencyResolver dep;
private SparkJLineCompletion completor;
/**
* completor - org.apache.spark.repl.SparkJLineCompletion (scala 2.10)
*/
private Object completor;
private Map<String, Object> binder;
private SparkVersion sparkVersion;
private File outputDir; // class outputdir for scala 2.11
private Object classServer; // classserver for scala 2.11
public SparkInterpreter(Properties property) {
@ -178,43 +193,91 @@ public class SparkInterpreter extends Interpreter {
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext"));
}
/**
* See org.apache.spark.sql.SparkSession.hiveClassesArePresent
* @return
*/
private boolean hiveClassesArePresent() {
try {
this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState");
this.getClass().forName("org.apache.spark.sql.hive.HiveSharedState");
this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf");
return true;
} catch (ClassNotFoundException | NoClassDefFoundError e) {
return false;
}
}
private boolean importImplicit() {
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.importImplicit"));
}
public SQLContext getSQLContext() {
public Object getSparkSession() {
synchronized (sharedInterpreterLock) {
if (sqlc == null) {
if (useHiveContext()) {
String name = "org.apache.spark.sql.hive.HiveContext";
Constructor<?> hc;
try {
hc = getClass().getClassLoader().loadClass(name)
.getConstructor(SparkContext.class);
sqlc = (SQLContext) hc.newInstance(getSparkContext());
} catch (NoSuchMethodException | SecurityException
| ClassNotFoundException | InstantiationException
| IllegalAccessException | IllegalArgumentException
| InvocationTargetException e) {
logger.warn("Can't create HiveContext. Fallback to SQLContext", e);
// when hive dependency is not loaded, it'll fail.
// in this case SQLContext can be used.
sqlc = new SQLContext(getSparkContext());
}
} else {
sqlc = new SQLContext(getSparkContext());
}
if (sparkSession == null) {
createSparkSession();
}
return sqlc;
return sparkSession;
}
}
public SQLContext getSQLContext() {
synchronized (sharedInterpreterLock) {
if (Utils.isSpark2()) {
return getSQLContext_2();
} else {
return getSQLContext_1();
}
}
}
/**
* Get SQLContext for spark 2.x
*/
private SQLContext getSQLContext_2() {
if (sqlc == null) {
sqlc = (SQLContext) Utils.invokeMethod(sparkSession, "wrapped");
if (sqlc == null) {
sqlc = (SQLContext) Utils.invokeMethod(sparkSession, "sqlContext");
}
}
return sqlc;
}
public SQLContext getSQLContext_1() {
if (sqlc == null) {
if (useHiveContext()) {
String name = "org.apache.spark.sql.hive.HiveContext";
Constructor<?> hc;
try {
hc = getClass().getClassLoader().loadClass(name)
.getConstructor(SparkContext.class);
sqlc = (SQLContext) hc.newInstance(getSparkContext());
} catch (NoSuchMethodException | SecurityException
| ClassNotFoundException | InstantiationException
| IllegalAccessException | IllegalArgumentException
| InvocationTargetException e) {
logger.warn("Can't create HiveContext. Fallback to SQLContext", e);
// when hive dependency is not loaded, it'll fail.
// in this case SQLContext can be used.
sqlc = new SQLContext(getSparkContext());
}
} else {
sqlc = new SQLContext(getSparkContext());
}
}
return sqlc;
}
public SparkDependencyResolver getDependencyResolver() {
if (dep == null) {
dep = new SparkDependencyResolver(intp,
sc,
getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
dep = new SparkDependencyResolver(
(Global) Utils.invokeMethod(intp, "global"),
(ClassLoader) Utils.invokeMethod(Utils.invokeMethod(intp, "classLoader"), "getParent"),
sc,
getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
}
return dep;
}
@ -231,18 +294,98 @@ public class SparkInterpreter extends Interpreter {
return (DepInterpreter) p;
}
/**
* Spark 2.x
* Create SparkSession
*/
public Object createSparkSession() {
logger.info("------ Create new SparkContext {} -------", getProperty("master"));
String execUri = System.getenv("SPARK_EXECUTOR_URI");
conf.setAppName(getProperty("spark.app.name"));
conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath());
if (execUri != null) {
conf.set("spark.executor.uri", execUri);
}
if (System.getenv("SPARK_HOME") != null) {
conf.setSparkHome(System.getenv("SPARK_HOME"));
}
conf.set("spark.scheduler.mode", "FAIR");
conf.setMaster(getProperty("master"));
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
if (!key.startsWith("spark.") || !val.trim().isEmpty()) {
logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val));
conf.set(key, val);
}
}
Class SparkSession = Utils.findClass("org.apache.spark.sql.SparkSession");
Object builder = Utils.invokeStaticMethod(SparkSession, "builder");
Utils.invokeMethod(builder, "config", new Class[]{ SparkConf.class }, new Object[]{ conf });
if (useHiveContext()) {
if (hiveClassesArePresent()) {
Utils.invokeMethod(builder, "enableHiveSupport");
sparkSession = Utils.invokeMethod(builder, "getOrCreate");
logger.info("Created Spark session with Hive support");
} else {
Utils.invokeMethod(builder, "config",
new Class[]{ String.class, String.class},
new Object[]{ "spark.sql.catalogImplementation", "in-memory"});
sparkSession = Utils.invokeMethod(builder, "getOrCreate");
logger.info("Created Spark session with Hive support");
}
} else {
sparkSession = Utils.invokeMethod(builder, "getOrCreate");
logger.info("Created Spark session");
}
return sparkSession;
}
public SparkContext createSparkContext() {
if (Utils.isSpark2()) {
return createSparkContext_2();
} else {
return createSparkContext_1();
}
}
/**
* Create SparkContext for spark 2.x
* @return
*/
private SparkContext createSparkContext_2() {
return (SparkContext) Utils.invokeMethod(sparkSession, "sparkContext");
}
public SparkContext createSparkContext_1() {
logger.info("------ Create new SparkContext {} -------", getProperty("master"));
String execUri = System.getenv("SPARK_EXECUTOR_URI");
String[] jars = SparkILoop.getAddedJars();
String[] jars = null;
if (Utils.isScala2_10()) {
jars = (String[]) Utils.invokeStaticMethod(SparkILoop.class, "getAddedJars");
} else {
jars = (String[]) Utils.invokeStaticMethod(
Utils.findClass("org.apache.spark.repl.Main"), "getAddedJars");
}
String classServerUri = null;
try { // in case of spark 1.1x, spark 1.2x
Method classServer = interpreter.intp().getClass().getMethod("classServer");
HttpServer httpServer = (HttpServer) classServer.invoke(interpreter.intp());
classServerUri = httpServer.uri();
Method classServer = intp.getClass().getMethod("classServer");
Object httpServer = classServer.invoke(intp);
classServerUri = (String) Utils.invokeMethod(httpServer, "uri");
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
// continue
@ -250,8 +393,8 @@ public class SparkInterpreter extends Interpreter {
if (classServerUri == null) {
try { // for spark 1.3x
Method classServer = interpreter.intp().getClass().getMethod("classServerUri");
classServerUri = (String) classServer.invoke(interpreter.intp());
Method classServer = intp.getClass().getMethod("classServerUri");
classServerUri = (String) classServer.invoke(intp);
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
// continue instead of: throw new InterpreterException(e);
@ -261,10 +404,15 @@ public class SparkInterpreter extends Interpreter {
}
}
SparkConf conf =
new SparkConf()
.setMaster(getProperty("master"))
.setAppName(getProperty("spark.app.name"));
if (Utils.isScala2_11()) {
classServer = createHttpServer(outputDir);
Utils.invokeMethod(classServer, "start");
classServerUri = (String) Utils.invokeMethod(classServer, "uri");
}
conf.setMaster(getProperty("master"))
.setAppName(getProperty("spark.app.name"));
if (classServerUri != null) {
conf.set("spark.repl.class.uri", classServerUri);
@ -376,6 +524,7 @@ public class SparkInterpreter extends Interpreter {
@Override
public void open() {
conf = new SparkConf();
URL[] urls = getClassloaderUrls();
// Very nice discussion about how scala compiler handle classpath
@ -392,17 +541,49 @@ public class SparkInterpreter extends Interpreter {
* getClass.getClassLoader >> } >> in.setContextClassLoader()
*/
Settings settings = new Settings();
if (getProperty("args") != null) {
String[] argsArray = getProperty("args").split(" ");
LinkedList<String> argList = new LinkedList<String>();
for (String arg : argsArray) {
argList.add(arg);
// process args
String args = getProperty("args");
if (args == null) {
args = "";
}
String[] argsArray = args.split(" ");
LinkedList<String> argList = new LinkedList<String>();
for (String arg : argsArray) {
argList.add(arg);
}
if (Utils.isScala2_10()) {
scala.collection.immutable.List<String> list =
JavaConversions.asScalaBuffer(argList).toList();
Object sparkCommandLine = Utils.instantiateClass(
"org.apache.spark.repl.SparkCommandLine",
new Class[]{ scala.collection.immutable.List.class },
new Object[]{ list });
settings = (Settings) Utils.invokeMethod(sparkCommandLine, "settings");
} else {
String sparkReplClassDir = getProperty("spark.repl.classdir");
if (sparkReplClassDir == null) {
sparkReplClassDir = System.getProperty("spark.repl.classdir");
}
if (sparkReplClassDir == null) {
sparkReplClassDir = System.getProperty("java.io.tmpdir");
}
SparkCommandLine command =
new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer(
argList).toList());
settings = command.settings();
outputDir = createTempDir(sparkReplClassDir);
argList.add("-Yrepl-class-based");
argList.add("-Yrepl-outdir");
argList.add(outputDir.getAbsolutePath());
scala.collection.immutable.List<String> list =
JavaConversions.asScalaBuffer(argList).toList();
settings.processArguments(list, true);
}
// set classpath for scala compiler
@ -470,7 +651,19 @@ public class SparkInterpreter extends Interpreter {
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
System.setProperty("scala.repl.name.line", "line" + this.hashCode() + "$");
/* Required for scoped mode.
* In scoped mode multiple scala compiler (repl) generates class in the same directory.
* Class names is not randomly generated and look like '$line12.$read$$iw$$iw'
* Therefore it's possible to generated class conflict(overwrite) with other repl generated
* class.
*
* To prevent generated class name conflict,
* change prefix of generated class name from each scala compiler (repl) instance.
*
* In Spark 2.x, REPL generated wrapper class name should compatible with the pattern
* ^(\$line(?:\d+)\.\$read)(?:\$\$iw)+$
*/
System.setProperty("scala.repl.name.line", "$line" + this.hashCode());
// To prevent 'File name too long' error on some file system.
MutableSettings.IntSetting numClassFileSetting = settings.maxClassfileName();
@ -481,37 +674,45 @@ public class SparkInterpreter extends Interpreter {
synchronized (sharedInterpreterLock) {
/* create scala repl */
if (printREPLOutput()) {
this.interpreter = new SparkILoop(null, new PrintWriter(out));
this.interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out));
} else {
this.interpreter = new SparkILoop(null, new PrintWriter(Console.out(), false));
this.interpreter = new SparkILoop((java.io.BufferedReader) null,
new PrintWriter(Console.out(), false));
}
interpreter.settings_$eq(settings);
interpreter.createInterpreter();
intp = interpreter.intp();
intp.setContextClassLoader();
intp.initializeSynchronous();
intp = Utils.invokeMethod(interpreter, "intp");
Utils.invokeMethod(intp, "setContextClassLoader");
Utils.invokeMethod(intp, "initializeSynchronous");
if (classOutputDir == null) {
classOutputDir = settings.outputDirs().getSingleOutput().get();
} else {
// change SparkIMain class output dir
settings.outputDirs().setSingleOutput(classOutputDir);
ClassLoader cl = intp.classLoader();
try {
Field rootField = cl.getClass().getSuperclass().getDeclaredField("root");
rootField.setAccessible(true);
rootField.set(cl, classOutputDir);
} catch (NoSuchFieldException | IllegalAccessException e) {
logger.error(e.getMessage(), e);
if (Utils.isScala2_10()) {
if (classOutputDir == null) {
classOutputDir = settings.outputDirs().getSingleOutput().get();
} else {
// change SparkIMain class output dir
settings.outputDirs().setSingleOutput(classOutputDir);
ClassLoader cl = (ClassLoader) Utils.invokeMethod(intp, "classLoader");
try {
Field rootField = cl.getClass().getSuperclass().getDeclaredField("root");
rootField.setAccessible(true);
rootField.set(cl, classOutputDir);
} catch (NoSuchFieldException | IllegalAccessException e) {
logger.error(e.getMessage(), e);
}
}
completor = Utils.instantiateClass(
"org.apache.spark.repl.SparkJLineCompletion",
new Class[]{Utils.findClass("org.apache.spark.repl.SparkIMain")},
new Object[]{intp});
}
completor = new SparkJLineCompletion(intp);
if (Utils.isSpark2()) {
sparkSession = getSparkSession();
}
sc = getSparkContext();
if (sc.getPoolForName("fair").isEmpty()) {
Value schedulingMode = org.apache.spark.scheduler.SchedulingMode.FAIR();
@ -530,31 +731,50 @@ public class SparkInterpreter extends Interpreter {
z = new ZeppelinContext(sc, sqlc, null, dep,
Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
binder = (Map<String, Object>) getValue("_binder");
interpret("@transient val _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder;
if (Utils.isScala2_10()) {
binder = (Map<String, Object>) getValue("_binder");
} else {
binder = (Map<String, Object>) getLastObject();
}
binder.put("sc", sc);
binder.put("sqlc", sqlc);
binder.put("z", z);
binder.put("intp", intp);
intp.interpret("@transient val intp = _binder.get(\"intp\").asInstanceOf[org.apache.spark" +
".repl.SparkIMain]");
intp.interpret("@transient val z = "
if (Utils.isSpark2()) {
binder.put("spark", sparkSession);
}
interpret("@transient val z = "
+ "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]");
intp.interpret("@transient val sc = "
interpret("@transient val sc = "
+ "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]");
intp.interpret("@transient val sqlc = "
interpret("@transient val sqlc = "
+ "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]");
intp.interpret("@transient val sqlContext = "
interpret("@transient val sqlContext = "
+ "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]");
intp.interpret("import org.apache.spark.SparkContext._");
if (Utils.isSpark2()) {
interpret("@transient val spark = "
+ "_binder.get(\"spark\").asInstanceOf[org.apache.spark.sql.SparkSession]");
}
interpret("import org.apache.spark.SparkContext._");
if (importImplicit()) {
if (sparkVersion.oldSqlContextImplicits()) {
intp.interpret("import sqlContext._");
if (Utils.isSpark2()) {
interpret("import spark.implicits._");
interpret("import spark.sql");
interpret("import org.apache.spark.sql.functions._");
} else {
intp.interpret("import sqlContext.implicits._");
intp.interpret("import sqlContext.sql");
intp.interpret("import org.apache.spark.sql.functions._");
if (sparkVersion.oldSqlContextImplicits()) {
interpret("import sqlContext._");
} else {
interpret("import sqlContext.implicits._");
interpret("import sqlContext.sql");
interpret("import org.apache.spark.sql.functions._");
}
}
}
}
@ -570,18 +790,20 @@ public class SparkInterpreter extends Interpreter {
Integer.parseInt(getProperty("zeppelin.spark.maxResult")) + ")");
*/
try {
if (sparkVersion.oldLoadFilesMethodName()) {
Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class);
loadFiles.invoke(this.interpreter, settings);
} else {
Method loadFiles = this.interpreter.getClass().getMethod(
"org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
loadFiles.invoke(this.interpreter, settings);
if (Utils.isScala2_10()) {
try {
if (sparkVersion.oldLoadFilesMethodName()) {
Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class);
loadFiles.invoke(this.interpreter, settings);
} else {
Method loadFiles = this.interpreter.getClass().getMethod(
"org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
loadFiles.invoke(this.interpreter, settings);
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
}
// add jar from DepInterpreter
@ -625,6 +847,14 @@ public class SparkInterpreter extends Interpreter {
numReferenceOfSparkContext.incrementAndGet();
}
private Results.Result interpret(String line) {
return (Results.Result) Utils.invokeMethod(
intp,
"interpret",
new Class[] {String.class},
new Object[] {line});
}
private List<File> currentClassPath() {
List<File> paths = classPath(Thread.currentThread().getContextClassLoader());
String[] cps = System.getProperty("java.class.path").split(File.pathSeparator);
@ -664,17 +894,22 @@ public class SparkInterpreter extends Interpreter {
completionText = "";
cursor = completionText.length();
}
ScalaCompleter c = completor.completer();
Candidates ret = c.complete(completionText, cursor);
if (Utils.isScala2_10()) {
ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completor, "completor");
Candidates ret = c.complete(completionText, cursor);
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
}
return completions;
} else {
return new LinkedList<InterpreterCompletion>();
}
return completions;
}
private String getCompletionTargetString(String text, int cursor) {
@ -718,9 +953,15 @@ public class SparkInterpreter extends Interpreter {
return resultCompletionText;
}
/*
* this method doesn't work in scala 2.11
* Somehow intp.valueOfTerm returns scala.None always with -Yrepl-class-based option
*/
public Object getValue(String name) {
Object ret = intp.valueOfTerm(name);
if (ret instanceof None) {
Object ret = Utils.invokeMethod(
intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name});
if (ret instanceof None || ret instanceof scala.None$) {
return null;
} else if (ret instanceof Some) {
return ((Some) ret).get();
@ -729,6 +970,16 @@ public class SparkInterpreter extends Interpreter {
}
}
public Object getLastObject() {
IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest");
if (r == null || r.lineRep() == null) {
return null;
}
Object obj = r.lineRep().call("$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
return obj;
}
String getJobGroup(InterpreterContext context){
return "zeppelin-" + context.getParagraphId();
}
@ -808,7 +1059,7 @@ public class SparkInterpreter extends Interpreter {
scala.tools.nsc.interpreter.Results.Result res = null;
try {
res = intp.interpret(incomplete + s);
res = interpret(incomplete + s);
} catch (Exception e) {
sc.clearJobGroup();
out.setInterpreterOutput(null);
@ -831,8 +1082,8 @@ public class SparkInterpreter extends Interpreter {
// make sure code does not finish with comment
if (r == Code.INCOMPLETE) {
scala.tools.nsc.interpreter.Results.Result res;
res = intp.interpret(incomplete + "\nprint(\"\")");
scala.tools.nsc.interpreter.Results.Result res = null;
res = interpret(incomplete + "\nprint(\"\")");
r = getResultCode(res);
}
@ -849,12 +1100,23 @@ public class SparkInterpreter extends Interpreter {
}
private void putLatestVarInResourcePool(InterpreterContext context) {
String varName = intp.mostRecentVar();
String varName = (String) Utils.invokeMethod(intp, "mostRecentVar");
if (varName == null || varName.isEmpty()) {
return;
}
Object lastObj = getValue(varName);
Object lastObj = null;
try {
if (Utils.isScala2_10()) {
lastObj = getValue(varName);
} else {
lastObj = getLastObject();
}
} catch (NullPointerException e) {
// Some case, scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call throws an NPE
logger.error(e.getMessage(), e);
}
if (lastObj != null) {
ResourcePool resourcePool = context.getResourcePool();
resourcePool.put(context.getNoteId(), context.getParagraphId(),
@ -998,9 +1260,13 @@ public class SparkInterpreter extends Interpreter {
if (numReferenceOfSparkContext.decrementAndGet() == 0) {
sc.stop();
sc = null;
if (classServer != null) {
Utils.invokeMethod(classServer, "stop");
classServer = null;
}
}
intp.close();
Utils.invokeMethod(intp, "close");
}
@Override
@ -1025,4 +1291,56 @@ public class SparkInterpreter extends Interpreter {
public SparkVersion getSparkVersion() {
return sparkVersion;
}
private File createTempDir(String dir) {
File file = null;
// try Utils.createTempDir()
file = (File) Utils.invokeStaticMethod(
Utils.findClass("org.apache.spark.util.Utils"),
"createTempDir",
new Class[]{String.class, String.class},
new Object[]{dir, "spark"});
// fallback to old method
if (file == null) {
file = (File) Utils.invokeStaticMethod(
Utils.findClass("org.apache.spark.util.Utils"),
"createTempDir",
new Class[]{String.class},
new Object[]{dir});
}
return file;
}
private Object createHttpServer(File outputDir) {
SparkConf conf = new SparkConf();
try {
// try to create HttpServer
Constructor<?> constructor = getClass().getClassLoader()
.loadClass("org.apache.spark.HttpServer")
.getConstructor(new Class[]{
SparkConf.class, File.class, SecurityManager.class, int.class, String.class});
return constructor.newInstance(new Object[] {
conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"});
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e) {
// fallback to old constructor
Constructor<?> constructor = null;
try {
constructor = getClass().getClassLoader()
.loadClass("org.apache.spark.HttpServer")
.getConstructor(new Class[]{
File.class, SecurityManager.class, int.class, String.class});
return constructor.newInstance(new Object[] {
outputDir, new SecurityManager(conf), 0, "HTTP Server"});
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e1) {
logger.error(e1.getMessage(), e1);
return null;
}
}
}
}

2
spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
View file

@ -27,9 +27,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;

6
spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
View file

@ -32,10 +32,12 @@ public class SparkVersion {
public static final SparkVersion SPARK_1_4_0 = SparkVersion.fromVersionString("1.4.0");
public static final SparkVersion SPARK_1_5_0 = SparkVersion.fromVersionString("1.5.0");
public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0");
public static final SparkVersion SPARK_1_7_0 = SparkVersion.fromVersionString("1.7.0");
public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0");
public static final SparkVersion SPARK_2_1_0 = SparkVersion.fromVersionString("2.1.0");
public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_7_0;
public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_1_0;
private int version;
private String versionString;

101
spark/src/main/java/org/apache/zeppelin/spark/Utils.java Normal file
View file

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
/**
* Utility and helper functions for the Spark Interpreter
*/
class Utils {
public static Logger logger = LoggerFactory.getLogger(Utils.class);
static Object invokeMethod(Object o, String name) {
return invokeMethod(o, name, new Class[]{}, new Object[]{});
}
static Object invokeMethod(Object o, String name, Class[] argTypes, Object[] params) {
try {
return o.getClass().getMethod(name, argTypes).invoke(o, params);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
logger.error(e.getMessage(), e);
}
return null;
}
static Object invokeStaticMethod(Class c, String name, Class[] argTypes, Object[] params) {
try {
return c.getMethod(name, argTypes).invoke(null, params);
} catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
logger.error(e.getMessage(), e);
}
return null;
}
static Object invokeStaticMethod(Class c, String name) {
return invokeStaticMethod(c, name, new Class[]{}, new Object[]{});
}
static Class findClass(String name) {
try {
return Utils.class.forName(name);
} catch (ClassNotFoundException e) {
logger.error(e.getMessage(), e);
return null;
}
}
static Object instantiateClass(String name, Class[] argTypes, Object[] params) {
try {
Constructor<?> constructor = Utils.class.getClassLoader()
.loadClass(name).getConstructor(argTypes);
return constructor.newInstance(params);
} catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException |
InstantiationException | InvocationTargetException e) {
logger.error(e.getMessage(), e);
}
return null;
}
// function works after intp is initialized
static boolean isScala2_10() {
try {
Utils.class.forName("org.apache.spark.repl.SparkIMain");
return true;
} catch (ClassNotFoundException e) {
return false;
}
}
static boolean isScala2_11() {
return !isScala2_10();
}
static boolean isSpark2() {
try {
Utils.class.forName("org.apache.spark.sql.SparkSession");
return true;
} catch (ClassNotFoundException e) {
return false;
}
}
}

2
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -31,7 +31,6 @@ import java.util.List;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.catalyst.expressions.Attribute;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
@ -70,7 +69,6 @@ public class ZeppelinContext {
public SparkContext sc;
public SQLContext sqlContext;
public HiveContext hiveContext;
private GUI gui;
@ZeppelinApi

28
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java
View file

@ -29,7 +29,6 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.repl.SparkIMain;
import org.apache.zeppelin.dep.AbstractDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -60,7 +59,7 @@ import scala.tools.nsc.util.MergedClassPath;
public class SparkDependencyResolver extends AbstractDependencyResolver {
Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class);
private Global global;
private SparkIMain intp;
private ClassLoader runtimeClassLoader;
private SparkContext sc;
private final String[] exclusions = new String[] {"org.scala-lang:scala-library",
@ -71,11 +70,14 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
"org.apache.zeppelin:zeppelin-spark",
"org.apache.zeppelin:zeppelin-server"};
public SparkDependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath,
String additionalRemoteRepository) {
public SparkDependencyResolver(Global global,
ClassLoader runtimeClassLoader,
SparkContext sc,
String localRepoPath,
String additionalRemoteRepository) {
super(localRepoPath);
this.intp = intp;
this.global = intp.global();
this.global = global;
this.runtimeClassLoader = runtimeClassLoader;
this.sc = sc;
addRepoFromProperty(additionalRemoteRepository);
}
@ -127,24 +129,22 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException,
IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException {
ClassLoader cl = intp.classLoader().getParent();
Method addURL;
addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class});
addURL = runtimeClassLoader.getClass().getDeclaredMethod("addURL", new Class[] {URL.class});
addURL.setAccessible(true);
for (URL url : urls) {
addURL.invoke(cl, url);
addURL.invoke(runtimeClassLoader, url);
}
}
private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException,
IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException {
ClassLoader cl = intp.classLoader().getParent();
Method addURL;
addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class});
addURL = runtimeClassLoader.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class});
addURL.setAccessible(true);
for (URL url : urls) {
addURL.invoke(cl, url);
addURL.invoke(runtimeClassLoader, url);
}
}
@ -209,7 +209,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
private void loadFromFs(String artifact, boolean addSparkContext) throws Exception {
File jarFile = new File(artifact);
intp.global().new Run();
global.new Run();
if (sc.version().startsWith("1.1")) {
updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()});
@ -257,7 +257,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
+ artifactResult.getArtifact().getVersion());
}
intp.global().new Run();
global.new Run();
if (sc.version().startsWith("1.1")) {
updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0]));
} else {

15
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -29,7 +29,7 @@ from pyspark.broadcast import Broadcast
from pyspark.serializers import MarshalSerializer, PickleSerializer
# for back compatibility
from pyspark.sql import SQLContext, HiveContext, SchemaRDD, Row
from pyspark.sql import SQLContext, HiveContext, Row
class Logger(object):
def __init__(self):
@ -107,6 +107,7 @@ class PyZeppelinContext(dict):
class SparkVersion(object):
SPARK_1_4_0 = 140
SPARK_1_3_0 = 130
SPARK_2_0_0 = 200
def __init__(self, versionNumber):
self.version = versionNumber
@ -117,6 +118,9 @@ class SparkVersion(object):
def isImportAllPackageUnderSparkSql(self):
return self.version >= self.SPARK_1_3_0
def isSpark2(self):
return self.version >= self.SPARK_2_0_0
class PySparkCompletion:
def __init__(self, interpreterObject):
self.interpreterObject = interpreterObject
@ -175,6 +179,12 @@ sys.stderr = output
client = GatewayClient(port=int(sys.argv[1]))
sparkVersion = SparkVersion(int(sys.argv[2]))
if sparkVersion.isSpark2():
from pyspark.sql import SparkSession
else:
from pyspark.sql import SchemaRDD
if sparkVersion.isAutoConvertEnabled():
gateway = JavaGateway(client, auto_convert = True)
else:
@ -209,6 +219,9 @@ sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
sqlc = SQLContext(sc, intp.getSQLContext())
sqlContext = sqlc
if sparkVersion.isSpark2():
spark = SparkSession(sc, intp.getSparkSession())
completion = PySparkCompletion(intp)
z = PyZeppelinContext(intp.getZeppelinContext())

34
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -19,17 +19,18 @@ package org.apache.zeppelin.spark;
import static org.junit.Assert.*;
import java.io.BufferedReader;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import org.apache.spark.HttpServer;
import org.apache.spark.SecurityManager;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.repl.SparkILoop;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.resource.WellKnownResourceName;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
@ -41,6 +42,7 @@ import org.junit.Test;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.tools.nsc.interpreter.IMain;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class SparkInterpreterTest {
@ -139,6 +141,7 @@ public class SparkInterpreterTest {
assertEquals(InterpreterResult.Code.INCOMPLETE, incomplete.code());
assertTrue(incomplete.message().length() > 0); // expecting some error
// message
/*
* assertEquals(1, repl.getValue("a")); assertEquals(2, repl.getValue("b"));
* repl.interpret("val ver = sc.version");
@ -174,6 +177,17 @@ public class SparkInterpreterTest {
assertNotNull(SparkInterpreter.setupListeners(sc));
}
@Test
public void testCreateDataFrame() {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context);
repl.interpret("people.toDF.count", context);
assertEquals(new Long(4), context.getResourcePool().get(
context.getNoteId(),
context.getParagraphId(),
WellKnownResourceName.ZeppelinReplResult.toString()).get());
}
@Test
public void testSparkSql(){
repl.interpret("case class Person(name:String, age:Int)\n", context);
@ -182,15 +196,15 @@ public class SparkInterpreterTest {
if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default.
// create new interpreter
Properties p = new Properties();
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.open();
// create new interpreter
Properties p = new Properties();
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.open();
repl.interpret("case class Man(name:String, age:Int)", context);
repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context);
assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code());
repl2.getSparkContext().stop();
repl.interpret("case class Man(name:String, age:Int)", context);
repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context);
assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code());
repl2.getSparkContext().stop();
}
}

6
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -159,11 +159,13 @@ public class SparkSqlInterpreterTest {
repl.interpret(
"val raw = csv.map(_.split(\",\")).map(p => Row(p(0),toInt(p(1)),p(2)))",
context);
repl.interpret("val people = z.sqlContext.applySchema(raw, schema)",
context);
if (isDataFrameSupported()) {
repl.interpret("val people = z.sqlContext.createDataFrame(raw, schema)",
context);
repl.interpret("people.toDF.registerTempTable(\"people\")", context);
} else {
repl.interpret("val people = z.sqlContext.applySchema(raw, schema)",
context);
repl.interpret("people.registerTempTable(\"people\")", context);
}

1
spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java
View file

@ -19,7 +19,6 @@ package org.apache.zeppelin.spark.dep;
import static org.junit.Assert.assertEquals;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.junit.Test;
public class SparkDependencyResolverTest {

29
zeppelin-display/pom.xml
View file

@ -27,17 +27,12 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-display</artifactId>
<artifactId>zeppelin-display_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Display system apis</name>
<url>http://zeppelin.apache.org</url>
<properties>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
@ -86,16 +81,34 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.1.1</version>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencies>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
</dependency>
</dependencies>
</profile>
</profiles>
<build>
<plugins>
<plugin>

47
zeppelin-distribution/pom.xml
View file

@ -45,9 +45,37 @@
<!-- NOTE: These dependency declarations are only required to sort this project to the
end of the line in the multimodule build.
-->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scalap</artifactId>
<version>${scala.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<artifactId>zeppelin-server</artifactId>
<artifactId>zeppelin-server_2.10</artifactId>
<groupId>${project.groupId}</groupId>
<version>${project.version}</version>
</dependency>
@ -84,6 +112,23 @@
</build>
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>publish-distr</id>
<activation>

56
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -30,7 +30,7 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Maven scm api (org.apache.maven.scm:maven-scm-api:jar:1.4 - https://maven.apache.org/scm/)
(Apache 2.0) Apache Maven scm provider svnexe (org.apache.maven.scm:maven-scm-provider-svnexe:jar:1.4 - http://maven.apache.org/scm/maven-scm-providers/maven-scm-providers-svn/maven-scm-provider-svnexe/index.html/)
(Apache 2.0) Apache Maven scm provider common (org.apache.maven.scm:maven-scm-provider-svn-commons:jar:1.4 - http://maven.apache.org/scm/maven-scm-providers/maven-scm-providers-svn/maven-scm-provider-svn-commons/)
(Apache 2.0) Apache Spark (org.apache.spark:spark:1.5.1) - http://spark.apache.org
(Apache 2.0) Apache Spark (http://spark.apache.org)
(Apache 2.0) Apache Hadoop (http://hadoop.apache.org)
(Apache 2.0) Apache Avro (org.apache.avro:avro:1.7.7 - http://avro.apache.org)
(Apache 2.0) Apache Curator (org.apache.curator:curator:2.4.0 - http://curator.apache.org/)
@ -44,14 +44,14 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Thrift (http://thrift.apache.org/)
(Apache 2.0) Apache Lucene (https://lucene.apache.org/)
(Apache 2.0) Apache Zookeeper (org.apache.zookeeper:zookeeper:jar:3.4.5 - http://zookeeper.apache.org/)
(Apache 2.0) Chill (com.twitter:chill-java:jar:0.5.0 - https://github.com/twitter/chill/)
(Apache 2.0) Chill (com.twitter:chill-java:jar:0.8.0 - https://github.com/twitter/chill/)
(Apache 2.0) Codehaus Plexus (org.codehaus.plexus:plexus:jar:1.5.6 - https://codehaus-plexus.github.io/)
(Apache 2.0) findbugs jsr305 (com.google.code.findbugs:jsr305:jar:1.3.9 - http://findbugs.sourceforge.net/)
(Apache 2.0) Google Guava (com.google.guava:guava:15.0 - https://code.google.com/p/guava-libraries/)
(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-core:2.5.3 - https://github.com/FasterXML/jackson-core)
(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-annotations:2.5.0 - https://github.com/FasterXML/jackson-core)
(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-annotations:2.5.3 - https://github.com/FasterXML/jackson-core)
(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-databind:2.5.3 - https://github.com/FasterXML/jackson-core)
(Apache 2.0) javax.servlet (org.eclipse.jetty.orbit:javax.servlet:jar:3.0.0.v201112011016 - http://www.eclipse.org/jetty)
(Apache 2.0) javax.servlet (org.eclipse.jetty.orbit:javax.servlet:jar:3.1.0.v201112011016 - http://www.eclipse.org/jetty)
(Apache 2.0) Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/)
(Apache 2.0) Jackson (org.codehaus.jackson:jackson-core-asl:1.9.13 - http://jackson.codehaus.org/)
(Apache 2.0) Javassist (org.javassist:javassist:jar:3.18.1-GA:compile - http://jboss-javassist.github.io/javassist/)
@ -62,9 +62,9 @@ The following components are provided under Apache License.
(Apache 2.0) xml apis (xml-apis:xml-apis:jar:1.4.01 - http://xerces.apache.org/xml-commons/components/external)
(Apache 2.0) java-xmlbuilder (com.jamesmurty.utils:java-xmlbuilder:jar:1.0 - https://github.com/jmurty/java-xmlbuilder)
(Apache 2.0) compress-lzf (com.ning:compress-lzf:jar:1.0.3 - https://github.com/ning/compress) Copyright 2009-2010 Ning, Inc.
(Apache 2.0) Snappy-java (org.xerial.snappy:snappy-java:1.1.1.7 - https://github.com/xerial/snappy-java/)
(Apache 2.0) Snappy-java (org.xerial.snappy:snappy-java:1.1.2.4 - https://github.com/xerial/snappy-java/)
(Apache 2.0) lz4-java (net.jpountz.lz4:lz4:jar:1.3.0 - https://github.com/jpountz/lz4-java)
(Apache 2.0) RoaringBitmap (org.roaringbitmap:RoaringBitmap:jar:0.4.5 - https://github.com/lemire/RoaringBitmap)
(Apache 2.0) RoaringBitmap (org.roaringbitmap:RoaringBitmap:jar:0.5.11 - https://github.com/lemire/RoaringBitmap)
(Apache 2.0) json4s (org.json4s:json4s-ast_2.10:jar:3.2.10 - https://github.com/json4s/json4s)
(Apache 2.0) HPPC Collections (com.carrotsearch:hppc:0.7.1 - http://labs.carrotsearch.com/hppc.html/hppc)
(Apache 2.0) Jackson-dataformat-CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.2 - http://wiki.fasterxml.com/JacksonForCbor)
@ -73,7 +73,7 @@ The following components are provided under Apache License.
(Apache 2.0) json-flattener (com.github.wnameless:json-flattener:0.1.6 - https://github.com/wnameless/json-flattener)
(Apache 2.0) Spatial4J (com.spatial4j:spatial4j:0.4.1 - https://github.com/spatial4j/spatial4j)
(Apache 2.0) T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest)
(Apache 2.0) Netty (io.netty:netty:3.10.5.Final - http://netty.io/)
(Apache 2.0) Netty (io.netty:netty:3.8.0.Final - http://netty.io/)
(Apache 2.0) Lucene Common Analyzers (org.apache.lucene:lucene-analyzers-common:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-analyzers-common)
(Apache 2.0) Lucene Memory (org.apache.lucene:lucene-backward-codecs:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-backward-codecs)
(Apache 2.0) Lucene Core (org.apache.lucene:lucene-core:5.3.1 - http://lucene.apache.org/lucene-parent/lucene-core)
@ -93,13 +93,15 @@ The following components are provided under Apache License.
(Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org)
(Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org)
(Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org)
(Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.4.1 - https://github.com/google/protobuf/releases)
(Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.5.0 - https://github.com/google/protobuf/releases)
(Apache 2.0) Alluxio Shell (org.alluxio:alluxio-shell:1.0.0 - http://alluxio.org)
(Apache 2.0) Alluxio Servers (org.alluxio:alluxio-core-server:1.0.0 - http://alluxio.org)
(Apache 2.0) Alluxio Minicluster (org.alluxio:alluxio-minicluster:1.0.0 - http://alluxio.org)
(Apache 2.0) Alluxio Underfs Local (org.alluxio:alluxio-underfs-local:1.0.0 - http://alluxio.org)
(Apache 2.0) Microsoft Azure Storage Library for Java (com.microsoft.azure:azure-storage:4.0.0 - https://github.com/Azure/azure-storage-java)
(Apache 2.0) Roboto Font (https://github.com/google/roboto/)
(Apache 2.0) stream (com.clearspring.analytics:stream:2.7.0) - https://github.com/addthis/stream-lib/blob/v2.7.0/LICENSE.txt
(Apache 2.0) io.dropwizard.metrics:3.1.2 - https://github.com/dropwizard/metrics/blob/v3.1.2/LICENSE
========================================================================
@ -136,15 +138,16 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) slf4j-log4j12 v1.7.10 (org.slf4j:slf4j-log4j12:jar:1.7.10 - http://www.slf4j.org) - http://www.slf4j.org/license.html
(The MIT License) bcprov-jdk15on v1.51 (org.bouncycastle:bcprov-jdk15on:jar:1.51 - http://www.bouncycastle.org/java.html) - http://www.bouncycastle.org/licence.html
(The MIT License) AnchorJS (https://github.com/bryanbraun/anchorjs) - https://github.com/bryanbraun/anchorjs/blob/master/README.md#license
(The MIT License) moment-duration-format v1.3.0 (https://github.com/jsmreese/moment-duration-format) - https://github.com/jsmreese/moment-duration-format/blob/master/LICENSE
The following components are provided under the MIT License.
(The MIT License) Objenesis (org.objenesis:objenesis:2.1 - https://github.com/easymock/objenesis) - Copyright (c) 2006-2015 the original author and authors
(The MIT License) JCL 1.1.1 implemented over SLF4J (org.slf4j:jcl-over-slf4j:1.7.5 - http://www.slf4j.org)
(The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.5 - http://www.slf4j.org)
(The MIT License) JCL 1.1.1 implemented over SLF4J (org.slf4j:jcl-over-slf4j:1.7.16 - http://www.slf4j.org)
(The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.16 - http://www.slf4j.org)
(The MIT License) angular-resource (angular-resource - https://github.com/angular/angular.js/tree/master/src/ngResource)
(The MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.4 - https://github.com/ralfstx/minimal-json)
(The MIT License) pyrolite (net.razorvine:pyrolite:4.9) - https://github.com/irmen/Pyrolite/blob/v4.9/LICENSE
========================================================================
BSD-style licenses
@ -167,20 +170,31 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
The following components are provided under the BSD-style License.
(New BSD License) JGit (org.eclipse.jgit:org.eclipse.jgit:jar:4.1.1.201511131810-r - https://eclipse.org/jgit/)
(New BSD License) Kryo (com.esotericsoftware.kryo:kryo:2.21 - http://code.google.com/p/kryo/)
(New BSD License) MinLog (com.esotericsoftware.minlog:minlog:1.2 - http://code.google.com/p/minlog/)
(New BSD License) Kryo (com.esotericsoftware.kryo:kryo:3.0.3 - http://code.google.com/p/kryo/)
(New BSD License) leveldbjni (org.fusesource.leveldbjni:leveldbjni-all:1.8) - https://github.com/fusesource/leveldbjni/blob/leveldbjni-1.8/license.txt
(New BSD License) MinLog (com.esotericsoftware.minlog:minlog:1.3 - http://code.google.com/p/minlog/)
(New BSD License) ReflectASM (com.esotericsoftware.reflectasm:reflectasm:1.07 - http://code.google.com/p/reflectasm/)
(BSD-like) Scala Library (org.scala-lang:scala-library:2.10.4 - http://www.scala-lang.org/)
(BSD-like) Scalap (org.scala-lang:scalap:2.10.4 - http://www.scala-lang.org/)
(BSD-like) (The BSD License) jline (org.scala-lang:jline:2.10.4 - http://www.scala-lang.org/)
(BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.10.4 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.10.4 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.10.4 - http://www.scala-lang.org/)
(BSD-like) Scala Library (org.scala-lang:scala-library:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scalap (org.scala-lang:scalap:2.11.7 - http://www.scala-lang.org/)
(BSD-like) (The BSD License) jline (org.scala-lang:jline:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.7 - http://www.scala-lang.org/)
(BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.7 - http://www.scala-lang.org/)
(BSD-like) ASM (asm:asm:jar:3.1 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
(New BSD License) Py4J (net.sf.py4j:py4j:0.9 - http://py4j.sourceforge.net/)
(New BSD License) Py4J (net.sf.py4j:py4j:0.10.1 - http://py4j.sourceforge.net/) - https://github.com/bartdag/py4j/blob/0.10.1/LICENSE.txt
(New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
(BSD 3 Clause) Paranamer (com.thoughtworks.paranamer:paranamer:jar:2.6) - https://github.com/paul-hammant/paranamer/blob/paranamer-parent-2.6/LICENSE.txt
(BSD 3 Clause) netlib core (com.github.fommil.netlib:core:1.1.2 - https://github.com/fommil/netlib-java/core)
(BSD 3 Clause) JPMML-Model (org.jpmml:pmml-model:1.2.7 - https://github.com/jpmml/jpmml-model)
(BSD License) AntLR Parser Generator (antlr:antlr:2.7.7 - http://www.antlr.org/)
(BSD License) ANTLR 4.5.2-1 (org.antlr:antlr4:4.5.2-1 - http://wwww.antlr.org/)
(BSD licence) ANTLR ST4 4.0.4 (org.antlr:ST4:4.0.4 - http://www.stringtemplate.org)
(BSD licence) ANTLR StringTemplate (org.antlr:stringtemplate:3.2.1 - http://www.stringtemplate.org)
(BSD-style) spire (org.spire-math:spire_2.11:0.7.1 - http://spire-math.org)
(BSD-style) spire-macros (org.spire-math:spire-macros_2.11:0.7.1 - http://spire-math.org)
(The BSD License) Fortran to Java ARPACK (net.sourceforge.f2j:arpack_combined_all:0.1 - http://f2j.sourceforge.net)
(The BSD License) xmlenc Library (xmlenc:xmlenc:0.52 - http://xmlenc.sourceforge.net)
========================================================================
CDDL license

18
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
View file

@ -25,6 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;
/**
@ -59,6 +60,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
}
RemoteInterpreterManagedProcess(String intpRunner,
@ -103,7 +105,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
cmdLine.addArgument(localRepoDir, false);
executor = new DefaultExecutor();
executor.setStreamHandler(new PumpStreamHandler(new ProcessLogOutputStream(logger)));
watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
executor.setWatchdog(watchdog);
@ -163,4 +165,18 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
public boolean isRunning() {
return running;
}
private static class ProcessLogOutputStream extends LogOutputStream {
private Logger logger;
public ProcessLogOutputStream(Logger logger) {
this.logger = logger;
}
@Override
protected void processLine(String s, int i) {
this.logger.debug(s);
}
}
}

48
zeppelin-server/pom.xml
View file

@ -27,7 +27,7 @@
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-server</artifactId>
<artifactId>zeppelin-server_2.10</artifactId>
<packaging>jar</packaging>
<version>0.7.0-SNAPSHOT</version>
<name>Zeppelin: Server</name>
@ -43,19 +43,25 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.10.4</version>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>2.10.4</version>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scalap</artifactId>
<version>2.10.4</version>
<version>${scala.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -221,6 +227,19 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
@ -258,8 +277,8 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.10</artifactId>
<version>2.1.1</version>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
@ -393,6 +412,23 @@
</build>
<profiles>
<profile>
<id>scala-2.11</id>
<activation>
<property><name>scala-2.11</name></property>
</activation>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_${scala.binary.version}</artifactId>
<version>1.0.2</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>using-source-tree</id>
<activation>

129
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@ -32,36 +31,35 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import com.google.gson.Gson;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.dep.Repository;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest;
import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest;
import org.apache.zeppelin.server.JsonResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.RemoteRepository;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.dep.Repository;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest;
import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest;
import org.apache.zeppelin.server.JsonResponse;
/**
* Interpreter Rest API
*
*/
@Path("/interpreter")
@Produces("application/json")
public class InterpreterRestApi {
Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class);
private static final Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class);
private InterpreterFactory interpreterFactory;
Gson gson = new Gson();
public InterpreterRestApi() {
}
public InterpreterRestApi(InterpreterFactory interpreterFactory) {
@ -70,52 +68,39 @@ public class InterpreterRestApi {
/**
* List all interpreter settings
* @return
*/
@GET
@Path("setting")
@ZeppelinApi
public Response listSettings() {
List<InterpreterSetting> interpreterSettings = null;
List<InterpreterSetting> interpreterSettings;
interpreterSettings = interpreterFactory.get();
return new JsonResponse(Status.OK, "", interpreterSettings).build();
return new JsonResponse<>(Status.OK, "", interpreterSettings).build();
}
/**
* Add new interpreter setting
* @param message
* @return
* @throws IOException
* @throws InterpreterException
*
* @param message NewInterpreterSettingRequest
*/
@POST
@Path("setting")
@ZeppelinApi
public Response newSettings(String message) {
try {
NewInterpreterSettingRequest request = gson.fromJson(message,
NewInterpreterSettingRequest.class);
NewInterpreterSettingRequest request =
gson.fromJson(message, NewInterpreterSettingRequest.class);
Properties p = new Properties();
p.putAll(request.getProperties());
InterpreterSetting interpreterSetting = interpreterFactory.add(request.getName(),
request.getGroup(),
request.getDependencies(),
request.getOption(),
p);
logger.info("new setting created with {}", interpreterSetting.id());
return new JsonResponse(Status.CREATED, "", interpreterSetting).build();
} catch (InterpreterException e) {
InterpreterSetting interpreterSetting = interpreterFactory
.createNewSetting(request.getName(), request.getGroup(), request.getDependencies(),
request.getOption(), p);
logger.info("new setting created with {}", interpreterSetting.getId());
return new JsonResponse<>(Status.CREATED, "", interpreterSetting).build();
} catch (InterpreterException | IOException e) {
logger.error("Exception in InterpreterRestApi while creating ", e);
return new JsonResponse(
Status.NOT_FOUND,
e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
} catch (IOException | RepositoryException e) {
logger.error("Exception in InterpreterRestApi while creating ", e);
return new JsonResponse(
Status.INTERNAL_SERVER_ERROR,
e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e))
.build();
}
}
@ -126,26 +111,25 @@ public class InterpreterRestApi {
logger.info("Update interpreterSetting {}", settingId);
try {
UpdateInterpreterSettingRequest request = gson.fromJson(message,
UpdateInterpreterSettingRequest.class);
interpreterFactory.setPropertyAndRestart(settingId,
request.getOption(),
request.getProperties(),
request.getDependencies());
UpdateInterpreterSettingRequest request =
gson.fromJson(message, UpdateInterpreterSettingRequest.class);
interpreterFactory
.setPropertyAndRestart(settingId, request.getOption(), request.getProperties(),
request.getDependencies());
} catch (InterpreterException e) {
logger.error("Exception in InterpreterRestApi while updateSetting ", e);
return new JsonResponse(
Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
} catch (IOException | RepositoryException e) {
return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e))
.build();
} catch (IOException e) {
logger.error("Exception in InterpreterRestApi while updateSetting ", e);
return new JsonResponse(
Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
}
InterpreterSetting setting = interpreterFactory.get(settingId);
if (setting == null) {
return new JsonResponse(Status.NOT_FOUND, "", settingId).build();
return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build();
}
return new JsonResponse(Status.OK, "", setting).build();
return new JsonResponse<>(Status.OK, "", setting).build();
}
/**
@ -172,14 +156,14 @@ public class InterpreterRestApi {
interpreterFactory.restart(settingId);
} catch (InterpreterException e) {
logger.error("Exception in InterpreterRestApi while restartSetting ", e);
return new JsonResponse(
Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e))
.build();
}
InterpreterSetting setting = interpreterFactory.get(settingId);
if (setting == null) {
return new JsonResponse(Status.NOT_FOUND, "", settingId).build();
return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build();
}
return new JsonResponse(Status.OK, "", setting).build();
return new JsonResponse<>(Status.OK, "", setting).build();
}
/**
@ -188,27 +172,25 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
Map<String, RegisteredInterpreter> m = Interpreter.registeredInterpreters;
return new JsonResponse(Status.OK, "", m).build();
Map<String, InterpreterSetting> m = interpreterFactory.getAvailableInterpreterSettings();
return new JsonResponse<>(Status.OK, "", m).build();
}
/**
* List of dependency resolving repositories
* @return
*/
@GET
@Path("repository")
@ZeppelinApi
public Response listRepositories() {
List<RemoteRepository> interpreterRepositories = null;
interpreterRepositories = interpreterFactory.getRepositories();
return new JsonResponse(Status.OK, "", interpreterRepositories).build();
List<RemoteRepository> interpreterRepositories = interpreterFactory.getRepositories();
return new JsonResponse<>(Status.OK, "", interpreterRepositories).build();
}
/**
* Add new repository
* @param message
* @return
*
* @param message Repository
*/
@POST
@Path("repository")
@ -216,24 +198,21 @@ public class InterpreterRestApi {
public Response addRepository(String message) {
try {
Repository request = gson.fromJson(message, Repository.class);
interpreterFactory.addRepository(
request.getId(),
request.getUrl(),
request.isSnapshot(),
interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(),
request.getAuthentication());
logger.info("New repository {} added", request.getId());
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while adding repository ", e);
return new JsonResponse(
Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
}
return new JsonResponse(Status.CREATED).build();
}
/**
* Delete repository
* @param repoId
* @return
*
* @param repoId ID of repository
*/
@DELETE
@Path("repository/{repoId}")
@ -244,8 +223,8 @@ public class InterpreterRestApi {
interpreterFactory.removeRepository(repoId);
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while removing repository ", e);
return new JsonResponse(
Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build();
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
}
return new JsonResponse(Status.OK).build();
}

12
zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java
View file

@ -17,7 +17,6 @@
package org.apache.zeppelin.rest;
import org.apache.shiro.authc.*;
import org.apache.shiro.session.Session;
import org.apache.shiro.subject.Subject;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.server.JsonResponse;
@ -112,22 +111,15 @@ public class LoginRestApi {
LOG.warn(response.toString());
return response.build();
}
@POST
@Path("logout")
@ZeppelinApi
public Response logout() {
JsonResponse response;
Subject currentUser = org.apache.shiro.SecurityUtils.getSubject();
currentUser.logout();
Map<String, String> data = new HashMap<>();
data.put("principal", "anonymous");
data.put("roles", "");
data.put("ticket", "anonymous");
response = new JsonResponse(Response.Status.OK, "", data);
response = new JsonResponse(Response.Status.UNAUTHORIZED, "", "");
LOG.warn(response.toString());
return response.build();
}

270
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -18,8 +18,12 @@
package org.apache.zeppelin.rest;
import java.io.IOException;
import java.util.*;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@ -31,7 +35,14 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
@ -48,16 +59,7 @@ import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.socket.NotebookServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.utils.SecurityUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonReader;
import java.io.StringReader;
/**
* Rest api endpoint for the noteBook.
*/
@ -71,7 +73,8 @@ public class NotebookRestApi {
private SearchService notebookIndex;
private NotebookAuthorization notebookAuthorization;
public NotebookRestApi() {}
public NotebookRestApi() {
}
public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) {
this.notebook = notebook;
@ -87,21 +90,19 @@ public class NotebookRestApi {
@Path("{noteId}/permissions")
@ZeppelinApi
public Response getNotePermissions(@PathParam("noteId") String noteId) {
Note note = notebook.getNote(noteId);
HashMap<String, Set<String>> permissionsMap = new HashMap();
HashMap<String, Set<String>> permissionsMap = new HashMap<>();
permissionsMap.put("owners", notebookAuthorization.getOwners(noteId));
permissionsMap.put("readers", notebookAuthorization.getReaders(noteId));
permissionsMap.put("writers", notebookAuthorization.getWriters(noteId));
return new JsonResponse<>(Status.OK, "", permissionsMap).build();
}
String ownerPermissionError(Set<String> current,
Set<String> allowed) throws IOException {
private String ownerPermissionError(Set<String> current, Set<String> allowed) throws IOException {
LOG.info("Cannot change permissions. Connection owners {}. Allowed owners {}",
current.toString(), allowed.toString());
current.toString(), allowed.toString());
return "Insufficient privileges to change permissions.\n\n" +
"Allowed owners: " + allowed.toString() + "\n\n" +
"User belongs to: " + current.toString();
"Allowed owners: " + allowed.toString() + "\n\n" +
"User belongs to: " + current.toString();
}
/**
@ -112,25 +113,25 @@ public class NotebookRestApi {
@ZeppelinApi
public Response putNotePermissions(@PathParam("noteId") String noteId, String req)
throws IOException {
HashMap<String, HashSet> permMap = gson.fromJson(req,
new TypeToken<HashMap<String, HashSet>>(){}.getType());
/**
* TODO(jl): Fixed the type of HashSet
* https://issues.apache.org/jira/browse/ZEPPELIN-1162
*/
HashMap<String, HashSet> permMap =
gson.fromJson(req, new TypeToken<HashMap<String, HashSet>>() {
}.getType());
Note note = notebook.getNote(noteId);
String principal = SecurityUtils.getPrincipal();
HashSet<String> roles = SecurityUtils.getRoles();
LOG.info("Set permissions {} {} {} {} {}",
noteId,
principal,
permMap.get("owners"),
permMap.get("readers"),
permMap.get("writers")
);
LOG.info("Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"),
permMap.get("readers"), permMap.get("writers"));
HashSet<String> userAndRoles = new HashSet<String>();
HashSet<String> userAndRoles = new HashSet<>();
userAndRoles.add(principal);
userAndRoles.addAll(roles);
if (!notebookAuthorization.isOwner(noteId, userAndRoles)) {
return new JsonResponse<>(Status.FORBIDDEN, ownerPermissionError(userAndRoles,
notebookAuthorization.getOwners(noteId))).build();
return new JsonResponse<>(Status.FORBIDDEN,
ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))).build();
}
HashSet readers = permMap.get("readers");
@ -146,7 +147,7 @@ public class NotebookRestApi {
}
}
// Set writers, if owners is empty -> set to user requesting the change
if ( writers != null && !writers.isEmpty()) {
if (writers != null && !writers.isEmpty()) {
if (owners.isEmpty()) {
owners = Sets.newHashSet(SecurityUtils.getPrincipal());
}
@ -155,10 +156,8 @@ public class NotebookRestApi {
notebookAuthorization.setReaders(noteId, readers);
notebookAuthorization.setWriters(noteId, writers);
notebookAuthorization.setOwners(noteId, owners);
LOG.debug("After set permissions {} {} {}",
notebookAuthorization.getOwners(noteId),
notebookAuthorization.getReaders(noteId),
notebookAuthorization.getWriters(noteId));
LOG.debug("After set permissions {} {} {}", notebookAuthorization.getOwners(noteId),
notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId));
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
@ -167,13 +166,15 @@ public class NotebookRestApi {
/**
* bind a setting to note
*
* @throws IOException
*/
@PUT
@Path("interpreter/bind/{noteId}")
@ZeppelinApi
public Response bind(@PathParam("noteId") String noteId, String req) throws IOException {
List<String> settingIdList = gson.fromJson(req, new TypeToken<List<String>>(){}.getType());
List<String> settingIdList = gson.fromJson(req, new TypeToken<List<String>>() {
}.getType());
notebook.bindInterpretersToNote(noteId, settingIdList);
return new JsonResponse<>(Status.OK).build();
}
@ -185,38 +186,27 @@ public class NotebookRestApi {
@Path("interpreter/bind/{noteId}")
@ZeppelinApi
public Response bind(@PathParam("noteId") String noteId) {
List<InterpreterSettingListForNoteBind> settingList
= new LinkedList<InterpreterSettingListForNoteBind>();
List<InterpreterSettingListForNoteBind> settingList = new LinkedList<>();
List<InterpreterSetting> selectedSettings = notebook.getBindedInterpreterSettings(noteId);
for (InterpreterSetting setting : selectedSettings) {
settingList.add(new InterpreterSettingListForNoteBind(
setting.id(),
setting.getName(),
setting.getGroup(),
setting.getInterpreterInfos(),
true)
);
settingList.add(new InterpreterSettingListForNoteBind(setting.getId(), setting.getName(),
setting.getInterpreterInfos(), true));
}
List<InterpreterSetting> availableSettings = notebook.getInterpreterFactory().get();
for (InterpreterSetting setting : availableSettings) {
boolean selected = false;
for (InterpreterSetting selectedSetting : selectedSettings) {
if (selectedSetting.id().equals(setting.id())) {
if (selectedSetting.getId().equals(setting.getId())) {
selected = true;
break;
}
}
if (!selected) {
settingList.add(new InterpreterSettingListForNoteBind(
setting.id(),
setting.getName(),
setting.getGroup(),
setting.getInterpreterInfos(),
false)
);
settingList.add(new InterpreterSettingListForNoteBind(setting.getId(), setting.getName(),
setting.getInterpreterInfos(), false));
}
}
return new JsonResponse<>(Status.OK, "", settingList).build();
@ -228,7 +218,7 @@ public class NotebookRestApi {
public Response getNotebookList() throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false, subject);
return new JsonResponse<>(Status.OK, "", notesInfo ).build();
return new JsonResponse<>(Status.OK, "", notesInfo).build();
}
@GET
@ -245,8 +235,8 @@ public class NotebookRestApi {
/**
* export note REST API
*
* @param
*
* @param noteId ID of Note
* @return note JSON with status.OK
* @throws IOException
*/
@ -255,12 +245,12 @@ public class NotebookRestApi {
@ZeppelinApi
public Response exportNoteBook(@PathParam("id") String noteId) throws IOException {
String exportJson = notebook.exportNote(noteId);
return new JsonResponse(Status.OK, "", exportJson).build();
return new JsonResponse<>(Status.OK, "", exportJson).build();
}
/**
* import new note REST API
*
*
* @param req - notebook Json
* @return JSON with new note ID
* @throws IOException
@ -273,9 +263,10 @@ public class NotebookRestApi {
Note newNote = notebook.importNote(req, null, subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
/**
* Create new note REST API
*
* @param message - JSON with new note name
* @return JSON with new note ID
* @throws IOException
@ -284,9 +275,8 @@ public class NotebookRestApi {
@Path("/")
@ZeppelinApi
public Response createNote(String message) throws IOException {
LOG.info("Create new notebook by JSON {}" , message);
NewNotebookRequest request = gson.fromJson(message,
NewNotebookRequest.class);
LOG.info("Create new notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note note = notebook.createNote(subject);
List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
@ -307,12 +297,13 @@ public class NotebookRestApi {
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", note.getId() ).build();
return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
}
/**
* Delete note REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException
*/
@ -332,21 +323,21 @@ public class NotebookRestApi {
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.OK, "").build();
}
/**
* Clone note REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.CREATED
* @throws IOException, CloneNotSupportedException, IllegalArgumentException
*/
@POST
@Path("{notebookId}")
@ZeppelinApi
public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws
IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}" , message);
NewNotebookRequest request = gson.fromJson(message,
NewNotebookRequest.class);
public Response cloneNote(@PathParam("notebookId") String notebookId, String message)
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
String newNoteName = request.getName();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
@ -357,6 +348,7 @@ public class NotebookRestApi {
/**
* Insert paragraph REST API
*
* @param message - JSON containing paragraph's information
* @return JSON with status.OK
* @throws IOException
@ -370,7 +362,7 @@ public class NotebookRestApi {
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
@ -388,12 +380,13 @@ public class NotebookRestApi {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.CREATED, "", p.getId()).build();
return new JsonResponse<>(Status.CREATED, "", p.getId()).build();
}
/**
* Get paragraph REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with information of the paragraph
* @throws IOException
*/
@ -401,7 +394,7 @@ public class NotebookRestApi {
@Path("{notebookId}/paragraph/{paragraphId}")
@ZeppelinApi
public Response getParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId) throws IOException {
@PathParam("paragraphId") String paragraphId) throws IOException {
LOG.info("get paragraph {} {}", notebookId, paragraphId);
Note note = notebook.getNote(notebookId);
@ -414,11 +407,12 @@ public class NotebookRestApi {
return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
}
return new JsonResponse(Status.OK, "", p).build();
return new JsonResponse<>(Status.OK, "", p).build();
}
/**
* Move paragraph REST API
*
* @param newIndex - new index to move
* @return JSON with status.OK
* @throws IOException
@ -427,8 +421,8 @@ public class NotebookRestApi {
@Path("{notebookId}/paragraph/{paragraphId}/move/{newIndex}")
@ZeppelinApi
public Response moveParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId,
@PathParam("newIndex") String newIndex) throws IOException {
@PathParam("paragraphId") String paragraphId, @PathParam("newIndex") String newIndex)
throws IOException {
LOG.info("move paragraph {} {} {}", notebookId, paragraphId, newIndex);
Note note = notebook.getNote(notebookId);
@ -456,7 +450,8 @@ public class NotebookRestApi {
/**
* Delete paragraph REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException
*/
@ -464,7 +459,7 @@ public class NotebookRestApi {
@Path("{notebookId}/paragraph/{paragraphId}")
@ZeppelinApi
public Response deleteParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId) throws IOException {
@PathParam("paragraphId") String paragraphId) throws IOException {
LOG.info("delete paragraph {} {}", notebookId, paragraphId);
Note note = notebook.getNote(notebookId);
@ -487,36 +482,38 @@ public class NotebookRestApi {
/**
* Run notebook jobs REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("job/{notebookId}")
@ZeppelinApi
public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws
IOException, IllegalArgumentException {
public Response runNoteJobs(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
LOG.info("run notebook jobs {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
note.runAll();
return new JsonResponse<>(Status.OK).build();
}
/**
* Stop(delete) notebook jobs REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("job/{notebookId}")
@ZeppelinApi
public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws
IOException, IllegalArgumentException {
public Response stopNoteJobs(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
LOG.info("stop notebook jobs {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -530,18 +527,19 @@ public class NotebookRestApi {
}
return new JsonResponse<>(Status.OK).build();
}
/**
* Get notebook job status REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@GET
@Path("job/{notebookId}")
@ZeppelinApi
public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) throws
IOException, IllegalArgumentException {
public Response getNoteJobStatus(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
LOG.info("get notebook job status.");
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -550,23 +548,21 @@ public class NotebookRestApi {
return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build();
}
/**
* Run paragraph job REST API
*
*
* @param message - JSON with params if user wants to update dynamic form's value
* null, empty string, empty json if user doesn't want to update
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId,
String message) throws
IOException, IllegalArgumentException {
public Response runParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId, String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
@ -581,8 +577,8 @@ public class NotebookRestApi {
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request = gson.fromJson(message,
RunParagraphWithParametersRequest.class);
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
@ -598,16 +594,21 @@ public class NotebookRestApi {
/**
* Stop(delete) paragraph job REST API
* @param
*
* @param notebookId ID of Notebook
* @param paragraphId ID of Paragraph
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response stopParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId) throws
IOException, IllegalArgumentException {
public Response stopParagraph(@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException {
/**
* TODO(jl): Fixed notebookId to noteId
* https://issues.apache.org/jira/browse/ZEPPELIN-1163
*/
LOG.info("stop paragraph job {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
@ -621,9 +622,10 @@ public class NotebookRestApi {
p.abort();
return new JsonResponse<>(Status.OK).build();
}
/**
* Register cron job REST API
*
* @param message - JSON with cron expressions.
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
@ -631,18 +633,18 @@ public class NotebookRestApi {
@POST
@Path("cron/{notebookId}")
@ZeppelinApi
public Response registerCronJob(@PathParam("notebookId") String notebookId, String message) throws
IOException, IllegalArgumentException {
public Response registerCronJob(@PathParam("notebookId") String notebookId, String message)
throws IOException, IllegalArgumentException {
// TODO(jl): Fixed notebookId to noteId
LOG.info("Register cron job note={} request cron msg={}", notebookId, message);
CronRequest request = gson.fromJson(message,
CronRequest.class);
CronRequest request = gson.fromJson(message, CronRequest.class);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
if (!CronExpression.isValidExpression(request.getCronString())) {
return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build();
}
@ -651,60 +653,64 @@ public class NotebookRestApi {
config.put("cron", request.getCronString());
note.setConfig(config);
notebook.refreshCron(note.id());
return new JsonResponse<>(Status.OK).build();
}
/**
* Remove cron job REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("cron/{notebookId}")
@ZeppelinApi
public Response removeCronJob(@PathParam("notebookId") String notebookId) throws
IOException, IllegalArgumentException {
public Response removeCronJob(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
// TODO(jl): Fixed notebookId to noteId
LOG.info("Remove cron job note {}", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Map<String, Object> config = note.getConfig();
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
return new JsonResponse<>(Status.OK).build();
}
}
/**
* Get cron job REST API
* @param
*
* @param notebookId ID of Notebook
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@GET
@Path("cron/{notebookId}")
@ZeppelinApi
public Response getCronJob(@PathParam("notebookId") String notebookId) throws
IOException, IllegalArgumentException {
public Response getCronJob(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
// TODO(jl): Fixed notebookId to noteId
LOG.info("Get cron job note {}", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build();
}
/**
* Get notebook jobs for job manager
* @param
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@ -726,7 +732,7 @@ public class NotebookRestApi {
/**
* Get updated notebook jobs for job manager
* @param
*
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@ -734,8 +740,8 @@ public class NotebookRestApi {
@Path("jobmanager/{lastUpdateUnixtime}/")
@ZeppelinApi
public Response getUpdatedJobListforNotebook(
@PathParam("lastUpdateUnixtime") long lastUpdateUnixTime) throws
IOException, IllegalArgumentException {
@PathParam("lastUpdateUnixtime") long lastUpdateUnixTime)
throws IOException, IllegalArgumentException {
LOG.info("Get updated notebook jobs lastUpdateTime {}", lastUpdateUnixTime);
List<Map<String, Object>> notebookJobs;
@ -759,7 +765,7 @@ public class NotebookRestApi {
LOG.info("Searching notebooks for: {}", queryTerm);
String principal = SecurityUtils.getPrincipal();
HashSet<String> roles = SecurityUtils.getRoles();
HashSet<String> userAndRoles = new HashSet<String>();
HashSet<String> userAndRoles = new HashSet<>();
userAndRoles.add(principal);
userAndRoles.addAll(roles);
List<Map<String, String>> notebooksFound = notebookIndex.query(queryTerm);
@ -767,8 +773,8 @@ public class NotebookRestApi {
String[] Id = notebooksFound.get(i).get("id").split("/", 2);
String noteId = Id[0];
if (!notebookAuthorization.isOwner(noteId, userAndRoles) &&
!notebookAuthorization.isReader(noteId, userAndRoles) &&
!notebookAuthorization.isWriter(noteId, userAndRoles)) {
!notebookAuthorization.isReader(noteId, userAndRoles) &&
!notebookAuthorization.isWriter(noteId, userAndRoles)) {
notebooksFound.remove(i);
i--;
}

57
zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/InterpreterSettingListForNoteBind.java
View file

@ -19,69 +19,22 @@ package org.apache.zeppelin.rest.message;
import java.util.List;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterInfo;
/**
* InterpreterSetting information for binding
*/
public class InterpreterSettingListForNoteBind {
String id;
String name;
String group;
private String id;
private String name;
private boolean selected;
private List<InterpreterSetting.InterpreterInfo> interpreters;
private List<InterpreterInfo> interpreters;
public InterpreterSettingListForNoteBind(String id, String name,
String group,
List<InterpreterSetting.InterpreterInfo> interpreters,
boolean selected) {
super();
List<InterpreterInfo> interpreters, boolean selected) {
this.id = id;
this.name = name;
this.group = group;
this.interpreters = interpreters;
this.selected = selected;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getGroup() {
return group;
}
public void setGroup(String group) {
this.group = group;
}
public List<InterpreterSetting.InterpreterInfo> getInterpreterNames() {
return interpreters;
}
public void setInterpreterNames(List<InterpreterSetting.InterpreterInfo> interpreters) {
this.interpreters = interpreters;
}
public boolean isSelected() {
return selected;
}
public void setSelected(boolean selected) {
this.selected = selected;
}
}

13
zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java
View file

@ -24,16 +24,15 @@ import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.interpreter.InterpreterOption;
/**
* NewInterpreterSetting rest api request message
*
* NewInterpreterSetting rest api request message
*/
public class NewInterpreterSettingRequest {
String name;
String group;
private String name;
private String group;
Map<String, String> properties;
List<Dependency> dependencies;
InterpreterOption option;
private Map<String, String> properties;
private List<Dependency> dependencies;
private InterpreterOption option;
public NewInterpreterSettingRequest() {

39
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -45,6 +45,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.scheduler.Job;
@ -220,6 +222,12 @@ public class NotebookServer extends WebSocketServlet implements
case CHECKPOINT_NOTEBOOK:
checkpointNotebook(conn, notebook, messagereceived);
break;
case LIST_REVISION_HISTORY:
listRevisionHistory(conn, notebook, messagereceived);
break;
case NOTE_REVISION:
getNoteRevision(conn, notebook, messagereceived);
break;
case LIST_NOTEBOOK_JOBS:
unicastNotebookJobInfo(conn, messagereceived);
break;
@ -494,6 +502,8 @@ public class NotebookServer extends WebSocketServlet implements
addConnectionToNote(note.id(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
} else {
conn.send(serializeMessage(new Message(OP.NOTE).put("note", null)));
}
}
@ -1124,7 +1134,34 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = (String) fromMessage.get("noteId");
String commitMessage = (String) fromMessage.get("commitMessage");
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook.checkpointNote(noteId, commitMessage, subject);
Revision revision = notebook.checkpointNote(noteId, commitMessage, subject);
if (revision != null) {
List<NotebookRepo.Revision> revisions = notebook.listRevisionHistory(noteId, subject);
conn.send(serializeMessage(new Message(OP.LIST_REVISION_HISTORY)
.put("revisionList", revisions)));
}
}
private void listRevisionHistory(NotebookSocket conn, Notebook notebook,
Message fromMessage) throws IOException {
String noteId = (String) fromMessage.get("noteId");
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
List<NotebookRepo.Revision> revisions = notebook.listRevisionHistory(noteId, subject);
conn.send(serializeMessage(new Message(OP.LIST_REVISION_HISTORY)
.put("revisionList", revisions)));
}
private void getNoteRevision(NotebookSocket conn, Notebook notebook, Message fromMessage)
throws IOException {
String noteId = (String) fromMessage.get("noteId");
Revision revision = (Revision) fromMessage.get("revision");
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
Note revisionNote = notebook.getNoteRevision(noteId, revision, subject);
conn.send(serializeMessage(new Message(OP.NOTE_REVISION)
.put("noteId", noteId)
.put("revisionId", revision)
.put("data", revisionNote)));
}
/**

4
zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java
View file

@ -150,8 +150,10 @@ public class WebDriverManager {
firebugUrlString = "http://getfirebug.com/releases/firebug/1.11/firebug-1.11.4.xpi";
else if (firefoxVersion >= 23 && firefoxVersion < 30)
firebugUrlString = "http://getfirebug.com/releases/firebug/1.12/firebug-1.12.8.xpi";
else if (firefoxVersion >= 30)
else if (firefoxVersion >= 30 && firefoxVersion < 33)
firebugUrlString = "http://getfirebug.com/releases/firebug/2.0/firebug-2.0.7.xpi";
else if (firefoxVersion >= 33)
firebugUrlString = "http://getfirebug.com/releases/firebug/2.0/firebug-2.0.17.xpi";
LOG.info("firebug version: " + firefoxVersion + ", will be downloaded to " + tempPath);

11
zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinITUtils.java
View file

@ -20,6 +20,8 @@ package org.apache.zeppelin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.openqa.selenium.WebDriver;
import java.util.concurrent.TimeUnit;
public class ZeppelinITUtils {
@ -46,4 +48,13 @@ public class ZeppelinITUtils {
//wait for server to start.
sleep(5000, false);
}
public static void turnOffImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(0, TimeUnit.SECONDS);
}
public static void turnOnImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(AbstractZeppelinIT.MAX_IMPLICIT_WAIT,
TimeUnit.SECONDS);
}
}

19
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -343,9 +343,12 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
String xpathToShowTitle = getParagraphXPath(1) + "//ul/li/a[@ng-click='showTitle()']";
String xpathToHideTitle = getParagraphXPath(1) + "//ul/li/a[@ng-click='hideTitle()']";
collector.checkThat("Before Show Title : The title field contains",
driver.findElement(By.xpath(xpathToTitle)).getText(),
CoreMatchers.equalTo(""));
ZeppelinITUtils.turnOffImplicitWaits(driver);
Integer titleElems = driver.findElements(By.xpath(xpathToTitle)).size();
collector.checkThat("Before Show Title : The title doesn't exist",
titleElems,
CoreMatchers.equalTo(0));
ZeppelinITUtils.turnOnImplicitWaits(driver);
clickAndWait(By.xpath(xpathToSettingIcon));
collector.checkThat("Before Show Title : The title option in option panel of paragraph is labeled as ",
@ -363,9 +366,13 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
CoreMatchers.equalTo("Hide title"));
clickAndWait(By.xpath(xpathToHideTitle));
collector.checkThat("After Hide Title : The title field contains",
driver.findElement(By.xpath(xpathToTitle)).getText(),
CoreMatchers.equalTo(""));
ZeppelinITUtils.turnOffImplicitWaits(driver);
titleElems = driver.findElements(By.xpath(xpathToTitle)).size();
collector.checkThat("After Hide Title : The title field is hidden",
titleElems,
CoreMatchers.equalTo(0));
ZeppelinITUtils.turnOnImplicitWaits(driver);
driver.findElement(By.xpath(xpathToSettingIcon)).click();
driver.findElement(By.xpath(xpathToShowTitle)).click();

73
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -22,12 +22,13 @@ import java.io.IOException;
import java.lang.ref.WeakReference;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
@ -37,9 +38,6 @@ import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.server.ZeppelinServer;
import org.hamcrest.Description;
@ -51,7 +49,6 @@ import org.slf4j.LoggerFactory;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import org.sonatype.aether.RepositoryException;
public abstract class AbstractTestRestApi {
@ -102,6 +99,30 @@ public abstract class AbstractTestRestApi {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../");
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_WAR.getVarName(), "../zeppelin-web/dist");
LOG.info("Staring test Zeppelin up...");
// exclude org.apache.zeppelin.rinterpreter.* for scala 2.11 test
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
String interpreters = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS);
String interpretersCompatibleWithScala211Test = null;
for (String intp : interpreters.split(",")) {
if (intp.startsWith("org.apache.zeppelin.rinterpreter")) {
continue;
}
if (interpretersCompatibleWithScala211Test == null) {
interpretersCompatibleWithScala211Test = intp;
} else {
interpretersCompatibleWithScala211Test += "," + intp;
}
}
System.setProperty(
ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(),
interpretersCompatibleWithScala211Test);
executor = Executors.newSingleThreadExecutor();
executor.submit(server);
long s = System.currentTimeMillis();
@ -125,7 +146,7 @@ public abstract class AbstractTestRestApi {
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
if (intpSetting.getGroup().equals("spark")) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
}
@ -138,12 +159,12 @@ public abstract class AbstractTestRestApi {
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
pySpark = true;
sparkR = true;
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id());
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
} else {
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
if (intpSetting.getGroup().equals("spark")) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
}
@ -156,7 +177,7 @@ public abstract class AbstractTestRestApi {
sparkR = true;
}
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id());
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
}
}
}
@ -205,7 +226,7 @@ public abstract class AbstractTestRestApi {
}
private static boolean isActiveSparkHome(File dir) {
if (dir.getName().matches("spark-[0-9\\.]+-bin-hadoop[0-9\\.]+")) {
if (dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+")) {
File pidDir = new File(dir, "run");
if (pidDir.isDirectory() && pidDir.listFiles().length > 0) {
return true;
@ -241,6 +262,8 @@ public abstract class AbstractTestRestApi {
}
LOG.info("Test Zeppelin terminated.");
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName());
}
}
@ -375,18 +398,6 @@ public abstract class AbstractTestRestApi {
};
}
//Create new Setting and return Setting ID
protected String createTempSetting(String tempName)
throws IOException, RepositoryException {
InterpreterSetting setting = ZeppelinServer.notebook.getInterpreterFactory()
.add(tempName,
"newGroup",
new LinkedList<Dependency>(),
new InterpreterOption(false),
new Properties());
return setting.id();
}
protected TypeSafeMatcher<? super JsonElement> hasRootElementNamed(final String memberName) {
return new TypeSafeMatcher<JsonElement>() {
@Override
@ -407,6 +418,22 @@ public abstract class AbstractTestRestApi {
};
}
public static void ps() {
DefaultExecutor executor = new DefaultExecutor();
executor.setStreamHandler(new PumpStreamHandler(System.out, System.err));
CommandLine cmd = CommandLine.parse("ps");
cmd.addArgument("aux", false);
try {
executor.execute(cmd);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
/** Status code matcher */
protected Matcher<? super HttpMethodBase> isForbiden() { return responsesWith(403); }

5
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -25,7 +25,6 @@ import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
@ -69,7 +68,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> body = (Map<String, Object>) resp.get("body");
assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size());
assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(), body.size());
get.releaseConnection();
}
@ -162,7 +161,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// Call Restart Interpreter REST API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), "");
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
assertThat("test interpreter restart:", put, isAllowed());
put.releaseConnection();
break;

43
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -20,16 +20,10 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
@ -95,6 +89,17 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
int sparkVersion = getSparkVersionNumber(note);
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("spark")) {
ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId());
break;
}
}
// run markdown paragraph, again
Paragraph p = note.addParagraph();
Map config = p.getConfig();
@ -142,16 +147,22 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note");
int sparkVersion = getSparkVersionNumber(note);
int sparkVersionNumber = getSparkVersionNumber(note);
if (isPyspark() && sparkVersion >= 14) { // auto_convert enabled from spark 1.4
if (isPyspark() && sparkVersionNumber >= 14) { // auto_convert enabled from spark 1.4
// run markdown paragraph, again
Paragraph p = note.addParagraph();
Map config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
String sqlContextName = "sqlContext";
if (sparkVersionNumber >= 20) {
sqlContextName = "spark";
}
p.setText("%pyspark\nfrom pyspark.sql.functions import *\n"
+ "print(sqlContext.range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())");
+ "print(" + sqlContextName + ".range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())");
// p.getRepl("org.apache.zeppelin.spark.SparkInterpreter").open();
note.run(p.getId());
waitForFinish(p);
@ -197,15 +208,16 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
public void pySparkDepLoaderTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
int sparkVersionNumber = getSparkVersionNumber(note);
if (isPyspark() && getSparkVersionNumber(note) >= 14) {
if (isPyspark() && sparkVersionNumber >= 14) {
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
for (InterpreterSetting setting : settings) {
if (setting.getGroup().equals("spark")) {
ZeppelinServer.notebook.getInterpreterFactory().restart(setting.id());
if (setting.getName().equals("spark")) {
ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId());
break;
}
}
@ -227,9 +239,14 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
// load data using libraries from dep loader
Paragraph p1 = note.addParagraph();
p1.setConfig(config);
String sqlContextName = "sqlContext";
if (sparkVersionNumber >= 20) {
sqlContextName = "spark";
}
p1.setText("%pyspark\n" +
"from pyspark.sql import SQLContext\n" +
"print(sqlContext.read.format('com.databricks.spark.csv')" +
"print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" +
".load('"+ tmpFile.getAbsolutePath() +"').count())");
note.run(p1.getId());

36
zeppelin-web/.eslintrc Normal file
View file

@ -0,0 +1,36 @@
{
"env": {
"browser": true,
"jasmine": true,
"node": true
},
"globals": {
"angular": false,
"_": false,
"jQuery": false,
"hljs": false,
"confirm": false,
"alert": false,
"nv": false,
"ace": false,
"d3": false,
"BootstrapDialog": false,
"Handsontable": false,
"moment": false
},
"rules": {
"no-bitwise": 2,
"camelcase": 2,
"curly": 2,
"eqeqeq": 2,
"wrap-iife": [2, "any"],
"no-use-before-define": 0,
"new-cap": 2,
"no-caller": 2,
"quotes": [2, "single"],
"no-shadow": 0,
"no-undef": 2,
"no-unused-vars": [2, { "vars": "local", "args": "none" }],
"strict": [2, "global"]
}
}

38
zeppelin-web/.jshintrc
View file

@ -1,38 +0,0 @@
{
"node": true,
"browser": true,
"esnext": true,
"bitwise": true,
"camelcase": true,
"curly": true,
"eqeqeq": true,
"immed": true,
"indent": 2,
"latedef": true,
"newcap": true,
"noarg": true,
"quotmark": "single",
"shadow": "inner",
"regexp": true,
"undef": true,
"unused": "vars",
"strict": true,
"trailing": true,
"smarttabs": true,
"jasmine": true,
"predef": [ "inject" ],
"globals": {
"angular": false,
"_": false,
"jQuery": false,
"hljs": false,
"confirm": false,
"alert": false,
"nv": false,
"ace": false,
"d3": false,
"BootstrapDialog": false,
"Handsontable": false,
"moment": false
}
}

6
zeppelin-web/CONTRIBUTING.md
View file

@ -11,7 +11,7 @@ This will launch a Zeppelin WebApplication on port **9000** that will update on
Zeppelin WebApplication is using **AngularJS** as main Framework, and **Grunt** and **Bower** as helpers.
So you might want to get familiar with it.
So you might want to get familiar with it.
[Here is a good start](http://www.sitepoint.com/kickstart-your-angularjs-development-with-yeoman-grunt-and-bower/)
(There is obviously plenty more ressources to learn)
@ -21,7 +21,7 @@ So you might want to get familiar with it.
* We use a 2 spaces indentation
* We use single quotes
But don't worry, JSHint will make you remember it for the most part.
But don't worry, Eslint and Jscs will make you remember it for the most part.
We try not to have **JQuery except in directives**, If you want to include a library,
please search for its **angularJS** directive first.
@ -119,7 +119,7 @@ The file index.html will automatically update with the new bower_component
<br/>
**Example**: `./bower install angular-nvd3`
**Example**: `./bower install angular-nvd3`
You should find that line in the index.html file
```

18
zeppelin-web/Gruntfile.js
View file

@ -127,7 +127,7 @@ module.exports = function(grunt) {
'<%= yeoman.app %>/app/**/*.js',
'<%= yeoman.app %>/components/**/*.js'
],
tasks: ['newer:jshint:all', 'newer:jscs:all'],
tasks: ['newer:eslint:all', 'newer:jscs:all'],
options: {
livereload: '<%= connect.options.livereload %>'
}
@ -140,7 +140,7 @@ module.exports = function(grunt) {
},
jsTest: {
files: ['test/spec/{,*/}*.js'],
tasks: ['newer:jshint:test', 'newer:jscs:test', 'karma']
tasks: ['newer:eslint:test', 'newer:jscs:test', 'karma']
},
styles: {
files: [
@ -234,13 +234,7 @@ module.exports = function(grunt) {
}
},
// Make sure code styles are up to par and there are no obvious mistakes
jshint: {
options: {
jshintrc: '.jshintrc',
reporterOutput: '',
reporter: require('jshint-stylish')
},
eslint: {
all: {
src: [
'Gruntfile.js',
@ -250,7 +244,9 @@ module.exports = function(grunt) {
},
test: {
options: {
jshintrc: 'test/.jshintrc'
rules: {
'no-undef': 0
}
},
src: ['test/spec/{,*/}*.js']
}
@ -524,7 +520,7 @@ module.exports = function(grunt) {
grunt.registerTask('build', [
'jscs',
'jshint:all',
'eslint',
'htmlhint',
'clean:dist',
'wiredep',

3
zeppelin-web/bower.json
View file

@ -30,7 +30,8 @@
"ngtoast": "~2.0.0",
"ng-focus-if": "~1.0.2",
"bootstrap3-dialog": "bootstrap-dialog#~1.34.7",
"handsontable": "~0.24.2"
"handsontable": "~0.24.2",
"moment-duration-format": "^1.3.0"
},
"devDependencies": {
"angular-mocks": "1.5.0"

5
zeppelin-web/package.json
View file

@ -18,12 +18,13 @@
"grunt-contrib-copy": "^0.5.0",
"grunt-contrib-cssmin": "^0.9.0",
"grunt-contrib-htmlmin": "^0.3.0",
"grunt-contrib-jshint": "^0.10.0",
"grunt-contrib-uglify": "^0.4.0",
"grunt-contrib-watch": "^0.6.1",
"grunt-eslint": "^18.1.0",
"grunt-filerev": "^0.2.1",
"grunt-goog-webfont-dl": "^0.1.2",
"grunt-htmlhint": "^0.9.13",
"grunt-jscs": "^2.1.0",
"grunt-karma": "~0.8.3",
"grunt-newer": "^0.7.0",
"grunt-ng-annotate": "^0.10.0",
@ -31,8 +32,6 @@
"grunt-svgmin": "^0.4.0",
"grunt-usemin": "^2.1.1",
"grunt-wiredep": "~2.0.0",
"grunt-jscs": "^2.1.0",
"jshint-stylish": "^0.2.0",
"karma": "~0.12.23",
"karma-coverage": "^0.5.1",
"karma-jasmine": "~0.1.5",

2
zeppelin-web/pom.xml
View file

@ -60,7 +60,7 @@
<exclude>.bowerrc</exclude>
<exclude>.editorconfig</exclude>
<exclude>.jscsrc</exclude>
<exclude>.jshintrc</exclude>
<exclude>.eslintrc</exclude>
<exclude>.tmp/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>

3
zeppelin-web/src/app/app.controller.js
View file

@ -44,4 +44,7 @@ angular.module('zeppelinWebApp').controller('MainCtrl', function($scope, $rootSc
BootstrapDialog.defaultOptions.onshown = function() {
angular.element('#' + this.id).find('.btn:last').focus();
};
// Remove BootstrapDialog animation
BootstrapDialog.configDefaultOptions({animate: false});
});

9
zeppelin-web/src/app/app.js
View file

@ -98,7 +98,6 @@
var baseUrlSrv = angular.injector(['zeppelinWebApp']).get('baseUrlSrv');
// withCredentials when running locally via grunt
$http.defaults.withCredentials = true;
return $http.get(baseUrlSrv.getRestApiBase() + '/security/ticket').then(function(response) {
zeppelinWebApp.run(function($rootScope) {
$rootScope.ticket = angular.fromJson(response.data).body;
@ -109,11 +108,17 @@
}
function bootstrapApplication() {
zeppelinWebApp.run(function($rootScope, $location) {
$rootScope.$on('$routeChangeStart', function(event, next, current) {
if (!$rootScope.ticket && next.$$route && !next.$$route.publicAccess) {
$location.path('/');
}
});
});
angular.bootstrap(document, ['zeppelinWebApp']);
}
angular.element(document).ready(function() {
auth().then(bootstrapApplication);
});
}());

14
zeppelin-web/src/app/configuration/configuration.controller.js
View file

@ -13,8 +13,8 @@
*/
'use strict';
angular.module('zeppelinWebApp').controller('ConfigurationCtrl', function($scope, $route, $routeParams, $location,
$rootScope, $http, baseUrlSrv) {
angular.module('zeppelinWebApp').controller('ConfigurationCtrl', function($scope, $rootScope, $http,
baseUrlSrv, ngToast) {
$scope.configrations = [];
$scope._ = _;
@ -24,6 +24,16 @@ angular.module('zeppelinWebApp').controller('ConfigurationCtrl', function($scope
$scope.configurations = data.body;
}).
error(function(data, status, headers, config) {
if (status === 401) {
ngToast.danger({
content: 'You don\'t have permission on this page',
verticalPosition: 'bottom',
timeout: '3000'
});
setTimeout(function() {
window.location.replace('/');
}, 3000);
}
console.log('Error %o %o', status, data.message);
});
};

14
zeppelin-web/src/app/credential/credential.controller.js
View file

@ -1,4 +1,3 @@
/* jshint loopfunc: true */
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,8 +13,7 @@
*/
'use strict';
angular.module('zeppelinWebApp').controller('CredentialCtrl', function($scope, $route, $routeParams, $location,
$rootScope, $http, baseUrlSrv, ngToast) {
angular.module('zeppelinWebApp').controller('CredentialCtrl', function($scope, $rootScope, $http, baseUrlSrv, ngToast) {
$scope._ = _;
$scope.credentialInfo = [];
@ -30,6 +28,16 @@ angular.module('zeppelinWebApp').controller('CredentialCtrl', function($scope, $
console.log('Success %o %o', status, $scope.credentialInfo);
}).
error(function(data, status, headers, config) {
if (status === 401) {
ngToast.danger({
content: 'You don\'t have permission on this page',
verticalPosition: 'bottom',
timeout: '3000'
});
setTimeout(function() {
window.location.replace('/');
}, 3000);
}
console.log('Error %o %o', status, data.message);
});
};

40
zeppelin-web/src/app/home/home.controller.js
View file

@ -14,7 +14,7 @@
'use strict';
angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, notebookListDataFactory, websocketMsgSrv,
$rootScope, arrayOrderingSrv, $http, baseUrlSrv) {
$rootScope, arrayOrderingSrv) {
var vm = this;
vm.notes = notebookListDataFactory;
vm.websocketMsgSrv = websocketMsgSrv;
@ -35,6 +35,27 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
initHome();
$scope.reloadNotebookList = function() {
websocketMsgSrv.reloadAllNotesFromRepo();
$scope.isReloadingNotes = true;
};
$scope.toggleFolderNode = function(node) {
node.hidden = !node.hidden;
};
angular.element('#loginModal').on('hidden.bs.modal', function(e) {
$rootScope.$broadcast('initLoginValues');
});
/*
** $scope.$on functions below
*/
$scope.$on('setNoteMenu', function(event, notes) {
$scope.isReloadingNotes = false;
});
$scope.$on('setNoteContent', function(event, note) {
if (note) {
vm.note = note;
@ -53,21 +74,4 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
}
});
$scope.$on('setNoteMenu', function(event, notes) {
$scope.isReloadingNotes = false;
});
$scope.reloadNotebookList = function() {
websocketMsgSrv.reloadAllNotesFromRepo();
$scope.isReloadingNotes = true;
};
$scope.toggleFolderNode = function(node) {
node.hidden = !node.hidden;
};
angular.element('#loginModal').on('hidden.bs.modal', function(e) {
$rootScope.$broadcast('initLoginValues');
});
});

6
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -19,7 +19,7 @@ limitations under the License.
<h4>Create new interpreter</h4>
<div class="form-group" style="width:200px">
<b>Name</b>
<b>Interpreter Name</b>
<input id="newInterpreterSettingName" input pu-elastic-input
pu-elastic-input-minwidth="180px" ng-model="newInterpreterSetting.name" />
</div>
@ -29,8 +29,8 @@ limitations under the License.
style="width:180px">
<select class="form-control input-sm" ng-model="newInterpreterSetting.group"
ng-change="newInterpreterGroupChange()">
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'group'| orderBy: 'group'" value="{{availableInterpreter.group}}">
{{availableInterpreter.group}}
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'name'| orderBy: 'name'" value="{{availableInterpreter.name}}">
{{availableInterpreter.name}}
</option>
</select>
</div>

45
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -1,4 +1,3 @@
/* jshint loopfunc: true */
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,7 +14,7 @@
'use strict';
angular.module('zeppelinWebApp').controller('InterpreterCtrl',
function($scope, $route, $routeParams, $location, $rootScope, $http, baseUrlSrv, ngToast) {
function($scope, $http, baseUrlSrv, ngToast, $timeout, $route) {
var interpreterSettingsTmp = [];
$scope.interpreterSettings = [];
$scope.availableInterpreters = {};
@ -26,11 +25,39 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
var getInterpreterSettings = function() {
$http.get(baseUrlSrv.getRestApiBase() + '/interpreter/setting').success(function(data, status, headers, config) {
$scope.interpreterSettings = data.body;
checkDownloadingDependencies();
}).error(function(data, status, headers, config) {
if (status === 401) {
ngToast.danger({
content: 'You don\'t have permission on this page',
verticalPosition: 'bottom',
timeout: '3000'
});
setTimeout(function() {
window.location.replace('/');
}, 3000);
}
console.log('Error %o %o', status, data.message);
});
};
var checkDownloadingDependencies = function() {
var isDownloading = false;
for (var setting = 0; setting < $scope.interpreterSettings.length; setting++) {
if ($scope.interpreterSettings[setting].status === 'DOWNLOADING_DEPENDENCIES') {
isDownloading = true;
break;
}
}
if (isDownloading) {
$timeout(function() {
if ($route.current.$$route.originalPath === '/interpreter') {
getInterpreterSettings();
}
}, 2000);
}
};
var getAvailableInterpreters = function() {
$http.get(baseUrlSrv.getRestApiBase() + '/interpreter').success(function(data, status, headers, config) {
$scope.availableInterpreters = data.body;
@ -140,6 +167,7 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
$scope.interpreterSettings[index] = data.body;
removeTMPSettings(index);
thisConfirm.close();
checkDownloadingDependencies();
})
.error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
@ -184,15 +212,14 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
};
$scope.newInterpreterGroupChange = function() {
var el = _.pluck(_.filter($scope.availableInterpreters, {'group': $scope.newInterpreterSetting.group}),
var el = _.pluck(_.filter($scope.availableInterpreters, {'name': $scope.newInterpreterSetting.group}),
'properties');
var properties = {};
for (var i = 0; i < el.length; i++) {
var intpInfo = el[i];
for (var key in intpInfo) {
properties[key] = {
value: intpInfo[key].defaultValue,
value: intpInfo[key],
description: intpInfo[key].description
};
}
@ -262,6 +289,7 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
$scope.resetNewInterpreterSetting();
getInterpreterSettings();
$scope.showAddNewSetting = false;
checkDownloadingDependencies();
}).error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
ngToast.danger({content: data.message, verticalPosition: 'bottom'});
@ -448,6 +476,13 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl',
}
};
$scope.showErrorMessage = function(setting) {
BootstrapDialog.show({
title: 'Error downloading dependencies',
message: setting.errorReason
});
};
var init = function() {
$scope.resetNewInterpreterSetting();
$scope.resetNewRepositorySetting();

1
zeppelin-web/src/app/interpreter/interpreter.filter.js
View file

@ -1,4 +1,3 @@
/* jshint loopfunc: true */
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

24
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -84,7 +84,7 @@ limitations under the License.
</div>
<div class="box width-full"
ng-repeat="setting in interpreterSettings | orderBy: 'group' | filter: searchInterpreter">
ng-repeat="setting in interpreterSettings | orderBy: 'name' | filter: searchInterpreter">
<div id="{{setting.name | lowercase}}">
<div class="row interpreter">
<div class="col-md-12">
@ -93,12 +93,32 @@ limitations under the License.
<span style="display:inline-block" ng-repeat="interpreter in setting.interpreterGroup"
title="{{interpreter.class}}">
<span ng-show="!$first">, </span>
%<span ng-show="!$parent.$first || $first">{{setting.group}}</span
%<span ng-show="!$parent.$first || $first">{{setting.name}}</span
><span ng-show="(!$parent.$first || $first) && !$first">.</span
><span ng-show="!$first">{{interpreter.name}}</span>
<span ng-show="$parent.$first && $first">(default)</span>
</span>
</small>
<small ng-switch="setting.status">
<small ng-switch-when="READY">
<i style="color: green; margin-right: 3px;" class="fa fa-circle"
tooltip="Ready">
</i>
</small>
<small ng-switch-when="ERROR">
<i style="color: red; cursor: pointer" class="fa fa-circle"
ng-click="showErrorMessage(setting)"
tooltip="Error downloading dependencies">
</i>
</small>
<small ng-switch-default>
<i style="color: blue" class="fa fa-spinner spinAnimation"
tooltip="Dependencies are downloading">
</i>
</small>
</small>
</h3>
<span style="float:right">
<button class="btn btn-default btn-xs"

62
zeppelin-web/src/app/jobmanager/jobmanager.controller.js
View file

@ -1,4 +1,3 @@
/*jshint loopfunc: true, unused:false */
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +15,38 @@
angular.module('zeppelinWebApp')
.controller('JobmanagerCtrl',
function($scope, $route, $routeParams, $location, $rootScope, $http, $q,
websocketMsgSrv, baseUrlSrv, $interval, $timeout, SaveAsService) {
function($scope, websocketMsgSrv, $interval) {
$scope.filterValueToName = function(filterValue) {
var index = _.findIndex($scope.ACTIVE_INTERPRETERS, {value: filterValue});
if ($scope.ACTIVE_INTERPRETERS[index].name !== undefined) {
return $scope.ACTIVE_INTERPRETERS[index].name;
} else {
return 'undefined';
}
};
$scope.init = function() {
$scope.jobInfomations = [];
$scope.JobInfomationsByFilter = $scope.jobInfomations;
websocketMsgSrv.getNotebookJobsList();
var refreshObj = $interval(function() {
if ($scope.lastJobServerUnixTime !== undefined) {
websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime);
}
}, 1000);
$scope.$on('$destroy', function() {
$interval.cancel(refreshObj);
websocketMsgSrv.unsubscribeJobManager();
});
};
/*
** $scope.$on functions below
*/
$scope.$on('setNotebookJobs', function(event, responseData) {
$scope.lastJobServerUnixTime = responseData.lastResponseUnixTime;
@ -63,31 +92,4 @@ angular.module('zeppelinWebApp')
}
});
});
$scope.filterValueToName = function(filterValue) {
var index = _.findIndex($scope.ACTIVE_INTERPRETERS, {value: filterValue});
if ($scope.ACTIVE_INTERPRETERS[index].name !== undefined) {
return $scope.ACTIVE_INTERPRETERS[index].name;
} else {
return 'undefined';
}
};
$scope.init = function() {
$scope.jobInfomations = [];
$scope.JobInfomationsByFilter = $scope.jobInfomations;
websocketMsgSrv.getNotebookJobsList();
var refreshObj = $interval(function() {
if ($scope.lastJobServerUnixTime !== undefined) {
websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime);
}
}, 1000);
$scope.$on('$destroy', function() {
$interval.cancel(refreshObj);
websocketMsgSrv.unsubscribeJobManager();
});
};
});

3
zeppelin-web/src/app/jobmanager/jobs/job.controller.js
View file

@ -1,4 +1,3 @@
/*jshint loopfunc: true, unused:false */
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,7 +14,7 @@
'use strict';
angular.module('zeppelinWebApp')
.controller('JobCtrl', function($scope, $rootScope, $http, baseUrlSrv) {
.controller('JobCtrl', function($scope) {
$scope.init = function(jobInformation) {
$scope.progressValue = 0;

Some files were not shown because too many files have changed in this diff Show more