Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' of github.com:apache/zeppelin into zeppelin-3092-remote-github-integration

Browse source
This commit is contained in:
Mohamed Magdy 2018-01-23 12:09:51 +01:00
commit afa5de1877
94 changed files with 14792 additions and 1684 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
.travis.yml
View file

@ -30,7 +30,7 @@ cache:
addons:
apt:
sources:
- r-packages-precise
- r-packages-trusty
packages:
- r-base-dev
@ -42,8 +42,8 @@ env:
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
dist: precise
- jdk: "openjdk7"
dist: trusty
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Run e2e tests (in zeppelin-web)
@ -53,14 +53,11 @@ matrix:
sudo: false
dist: trusty
jdk: "oraclejdk8"
env: PYTHON="2" WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
env: PYTHON="2" WEB_E2E="true" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
addons:
apt:
sources:
- r-packages-trusty
packages:
- google-chrome-stable
- r-base-dev
# Test core modules
# Several tests were excluded from this configuration due to the following issues:
@ -68,55 +65,59 @@ matrix:
# After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
- sudo: required
jdk: "oraclejdk8"
dist: precise
dist: trusty
addons:
firefox: "31.0"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk7"
dist: precise
env: PYTHON="2" TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
- jdk: "oraclejdk8"
dist: trusty
addons:
firefox: "31.0"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false"
# Test interpreter modules
- jdk: "oraclejdk7"
dist: precise
- jdk: "openjdk7"
dist: trusty
env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="install -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
# Test spark module for 2.2.0 with scala 2.11, livy
- jdk: "oraclejdk8"
dist: precise
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
dist: trusty
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
# Test spark module for 2.1.0 with scala 2.11, livy
- jdk: "oraclejdk7"
dist: precise
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
- jdk: "openjdk7"
dist: trusty
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
# Test spark module for 2.0.2 with scala 2.11
- jdk: "oraclejdk7"
dist: precise
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
- jdk: "oraclejdk8"
dist: trusty
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.10
- jdk: "oraclejdk7"
dist: precise
env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
- jdk: "openjdk7"
dist: trusty
env: PYTHON="3" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.11
- jdk: "oraclejdk7"
dist: precise
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
- jdk: "oraclejdk8"
dist: trusty
env: PYTHON="2" SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test python/pyspark with python 2, livy 0.2
- sudo: required
dist: precise
jdk: "oraclejdk7"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
dist: trusty
jdk: "openjdk7"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
# Test python/pyspark with python 3, livy 0.3
- sudo: required
dist: precise
jdk: "oraclejdk7"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
dist: trusty
jdk: "openjdk7"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop3 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
before_install:
# check files included in commit range, clear bower_components if a bower.json file has changed.

2
LICENSE
View file

@ -257,6 +257,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(Apache 2.0) Software under ./bigquery/* was developed at Google (http://www.google.com/). Licensed under the Apache v2.0 License.
(Apache 2.0) Roboto Font (https://github.com/google/roboto/)
(Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras)
(Apache 2.0) Nimbus JOSE+JWT (https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home)
(Apache 2.0) jarchivelib (https://github.com/thrau/jarchivelib)
========================================================================
BSD 3-Clause licenses

24
bin/interpreter.sh
View file

@ -16,6 +16,7 @@
# limitations under the License.
#
bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
@ -50,11 +51,6 @@ while getopts "hc:p:r:d:l:v:u:g:" o; do
;;
u)
ZEPPELIN_IMPERSONATE_USER="${OPTARG}"
if [[ -z "$ZEPPELIN_IMPERSONATE_CMD" ]]; then
ZEPPELIN_IMPERSONATE_RUN_CMD=`echo "ssh ${ZEPPELIN_IMPERSONATE_USER}@localhost" `
else
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
;;
g)
INTERPRETER_SETTING_NAME=${OPTARG}
@ -96,6 +92,15 @@ INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-${INTERPRETER_SETTING_NAME}-"
if [[ -z "$ZEPPELIN_IMPERSONATE_CMD" ]]; then
if [[ "${INTERPRETER_ID}" != "spark" || "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" == "false" ]]; then
ZEPPELIN_IMPERSONATE_RUN_CMD=`echo "ssh ${ZEPPELIN_IMPERSONATE_USER}@localhost" `
fi
else
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
fi
@ -195,7 +200,7 @@ fi
addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}"
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" && "${INTERPRETER_ID}" != "spark" ]]; then
suid="$(id -u ${ZEPPELIN_IMPERSONATE_USER})"
if [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
INTERPRETER_RUN_COMMAND=${ZEPPELIN_IMPERSONATE_RUN_CMD}" '"
@ -206,15 +211,12 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
fi
if [[ -n "${SPARK_SUBMIT}" ]]; then
if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
else
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
fi
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
else
INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_PORT}`
fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
INTERPRETER_RUN_COMMAND+="'"
fi

13
conf/shiro.ini.template
View file

@ -56,6 +56,19 @@ user3 = password4, role2
#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
#securityManager.realms = $zeppelinHubRealm
## A same for configuring Knox SSO Realm
#knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm
#knoxJwtRealm.providerUrl = https://domain.example.com/
#knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html
#knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout
#knoxJwtRealm.redirectParam = originalUrl
#knoxJwtRealm.cookieName = hadoop-jwt
#knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem
#
#knoxJwtRealm.groupPrincipalMapping = group.principal.mapping
#knoxJwtRealm.principalMapping = principal.mapping
#authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
### If caching of user is required then uncomment below lines

1
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -62,6 +62,7 @@
<li><a href="{{BASE_PATH}}/usage/other_features/personalized_mode.html">Personalized Mode</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/customizing_homepage.html">Customizing Zeppelin Homepage</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/notebook_actions.html">Notebook Actions</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/cron_scheduler.html">Cron Scheduler</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>REST API</span></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/interpreter.html">Interpreter API</a></li>

BIN
docs/assets/themes/zeppelin/img/docs-img/cron_scheduler_dialog_box.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 132 KiB

24
docs/setup/security/shiro_authentication.md
View file

@ -210,6 +210,30 @@ securityManager.realms = $zeppelinHubRealm
> Note: ZeppelinHub is not releated to Apache Zeppelin project.
### Knox SSO
[KnoxSSO](https://knox.apache.org/books/knox-0-13-0/dev-guide.html#KnoxSSO+Integration) provides an abstraction for integrating any number of authentication systems and SSO solutions and enables participating web applications to scale to those solutions more easily. Without the token exchange capabilities offered by KnoxSSO each component UI would need to integrate with each desired solution on its own.
To enable this, apply the following change in `conf/shiro.ini` under `[main]` section.
```
### A sample for configuring Knox JWT Realm
knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm
## Domain of Knox SSO
knoxJwtRealm.providerUrl = https://domain.example.com/
## Url for login
knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html
## Url for logout
knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout
knoxJwtRealm.redirectParam = originalUrl
knoxJwtRealm.cookieName = hadoop-jwt
knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem
knoxJwtRealm.groupPrincipalMapping = group.principal.mapping
knoxJwtRealm.principalMapping = principal.mapping
# This is required if KNOX SSO is enabled, to check if "knoxJwtRealm.cookieName" cookie was expired/deleted.
authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter
```
## Secure Cookie for Zeppelin Sessions (optional)
Zeppelin can be configured to set `HttpOnly` flag in the session cookie. With this configuration, Zeppelin cookies can
not be accessed via client side scripts thus preventing majority of Cross-site scripting (XSS) attacks.

52
docs/usage/other_features/cron_scheduler.md Normal file
View file

@ -0,0 +1,52 @@
---
layout: page
title: "Running a Notebook on a Given Schedule Automatically"
description: "You can run a notebook on a given schedule automatically by setting up a cron scheduler on the notebook."
group: usage/other_features
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Running a Notebook on a Given Schedule Automatically
<div id="toc"></div>
Apache Zeppelin provides a cron scheduler for each notebook. You can run a notebook on a given schedule automatically by setting up a cron scheduler on the notebook.
## Setting up a cron scheduler on a notebook
Click the clock icon on the tool bar and open a cron scheduler dialog box.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/cron_scheduler_dialog_box.png" />
There are the following items which you can input or set:
### Preset
You can set a cron schedule easily by clicking each option such as `1m` and `5m`. The login user is set as a cron executing user automatically. You can also clear the cron schedule settings by clicking `None`.
### Cron expression
You can set the cron schedule by filling in this form. Please see [Cron Trigger Tutorial](http://www.quartz-scheduler.org/documentation/quartz-2.2.x/tutorials/crontrigger) for the available cron syntax.
### Cron executing user
You can set the cron executing user by filling in this form and press the enter key.
### auto-restart interpreter on cron execution
When this checkbox is set to "on", the interpreters which are binded to the notebook are stopped automatically after the cron execution. This feature is useful if you want to release the interpreter resources after the cron execution.
> **Note**: A cron execution is skipped if one of the paragraphs is in a state of `RUNNING` or `PENDING` no matter whether it is executed automatically (i.e. by the cron scheduler) or manually by a user opening this notebook.

2
interpreter-parent/pom.xml
View file

@ -37,7 +37,7 @@
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>0.8.0-SNAPSHOT</version>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

28
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.livy;
import org.apache.commons.lang.StringUtils;
import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -26,7 +27,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
/**
* Livy SparkSQL Interpreter for Zeppelin.
*/
@ -166,15 +166,18 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
protected List<String> parseSQLOutput(String output) {
List<String> rows = new ArrayList<>();
String[] lines = output.split("\n");
// Get first line by breaking on \n. We can guarantee
// that \n marks the end of the first line, but not for
// subsequent lines (as it could be in the cells)
String firstLine = output.split("\n", 2)[0];
// at least 4 lines, even for empty sql output
// +---+---+
// | a| b|
// +---+---+
// +---+---+
// use the first line to determinte the position of feach cell
String[] tokens = StringUtils.split(lines[0], "\\+");
// use the first line to determine the position of each cell
String[] tokens = StringUtils.split(firstLine, "\\+");
// pairs keeps the start/end position of each cell. We parse it from the first row
// which use '+' as separator
List<Pair> pairs = new ArrayList<>();
@ -186,17 +189,26 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
pairs.add(new Pair(start, end));
}
for (String line : lines) {
// Use the header line to determine the position
// of subsequent lines
int lineStart = 0;
int lineEnd = firstLine.length();
while (lineEnd < output.length()) {
// Only match format "|....|"
// skip line like "+---+---+" and "only showing top 1 row"
if (line.matches("^\\|.*\\|$")) {
String line = output.substring(lineStart, lineEnd);
// Use the DOTALL regex mode to match newlines
if (line.matches("(?s)^\\|.*\\|$")) {
List<String> cells = new ArrayList<>();
for (Pair pair : pairs) {
// strip the blank space around the cell
cells.add(line.substring(pair.start, pair.end).trim());
// strip the blank space around the cell and escape the string
cells.add(escapeJavaScript(line.substring(pair.start, pair.end)).trim());
}
rows.add(StringUtils.join(cells, "\t"));
}
// Determine position of next line skipping newline
lineStart += firstLine.length() + 1;
lineEnd = lineStart + firstLine.length();
}
return rows;
}

44
livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
View file

@ -124,5 +124,49 @@ public class LivySQLInterpreterTest {
assertEquals(2, rows.size());
assertEquals("a", rows.get(0));
assertEquals("1", rows.get(1));
// sql output with 3 rows, 3 columns, showing "only showing top 3 rows" with a line break in the data
// +---+---+---+
// | a| b| c|
// +---+---+---+
// | 1a| 1b| 1c|
// | 2a| 2
// b| 2c|
// | 3a| 3b| 3c|
// +---+---+---+
// only showing top 3 rows
rows = sqlInterpreter.parseSQLOutput("+---+----+---+\n" +
"| a| b| c|\n" +
"+---+----+---+\n" +
"| 1a| 1b| 1c|\n" +
"| 2a| 2\nb| 2c|\n" +
"| 3a| 3b| 3c|\n" +
"+---+---+---+\n" +
"only showing top 3 rows");
assertEquals(4, rows.size());
assertEquals("a\tb\tc", rows.get(0));
assertEquals("1a\t1b\t1c", rows.get(1));
assertEquals("2a\t2\\nb\t2c", rows.get(2));
assertEquals("3a\t3b\t3c", rows.get(3));
// sql output with 2 rows and one containing a tab
// +---+---+
// | a| b|
// +---+---+
// | 1| \ta|
// | 2| 2b|
// +---+---+
rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"| 1| \ta|\n" +
"| 2| 2b|\n" +
"+---+---+");
assertEquals(3, rows.size());
assertEquals("a\tb", rows.get(0));
assertEquals("1\t\\ta", rows.get(1));
assertEquals("2\t2b", rows.get(2));
}
}

6
markdown/src/main/java/org/apache/zeppelin/markdown/PegdownParser.java
View file

@ -41,8 +41,10 @@ public class PegdownParser implements MarkdownParser {
@Override
public String render(String markdownText) {
String html = "";
String parsed = processor.markdownToHtml(markdownText);
String parsed;
synchronized (processor) {
parsed = processor.markdownToHtml(markdownText);
}
if (null == parsed) {
throw new RuntimeException("Cannot parse markdown text to HTML using pegdown");
}

39
markdown/src/test/java/org/apache/zeppelin/markdown/PegdownParserTest.java
View file

@ -19,6 +19,7 @@ package org.apache.zeppelin.markdown;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -26,10 +27,8 @@ import static org.apache.zeppelin.markdown.PegdownParser.wrapWithMarkdownClassDi
import static org.junit.Assert.assertThat;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.*;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -37,6 +36,9 @@ public class PegdownParserTest {
Logger logger = LoggerFactory.getLogger(PegdownParserTest.class);
Markdown md;
@Rule
public ErrorCollector collector = new ErrorCollector();
@Before
public void setUp() throws Exception {
Properties props = new Properties();
@ -50,6 +52,35 @@ public class PegdownParserTest {
md.close();
}
@Test
public void testMultipleThread() {
ArrayList<Thread> arrThreads = new ArrayList<Thread>();
for (int i = 0; i < 10; i++) {
Thread t = new Thread() {
public void run() {
String r1 = null;
try {
r1 = md.interpret("# H1", null).code().name();
} catch (Exception e) {
logger.error("testTestMultipleThread failed to interpret", e);
}
collector.checkThat("SUCCESS",
CoreMatchers.containsString(r1));
}
};
t.start();
arrThreads.add(t);
}
for (int i = 0; i < 10; i++) {
try {
arrThreads.get(i).join();
} catch (InterruptedException e) {
logger.error("testTestMultipleThread failed to join threads", e);
}
}
}
@Test
public void testHeader() {
InterpreterResult r1 = md.interpret("# H1", null);

14
pom.xml
View file

@ -52,6 +52,7 @@
<inceptionYear>2013</inceptionYear>
<modules>
<module>interpreter-parent</module>
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>zeppelin-display</module>
@ -91,9 +92,8 @@
<scalacheck.version>1.12.5</scalacheck.version>
<!-- frontend maven plugin related versions-->
<node.version>v6.9.1</node.version>
<yarn.version>v0.18.1</yarn.version>
<npm.version>4.2.0</npm.version>
<node.version>v8.9.3</node.version>
<npm.version>5.5.1</npm.version>
<plugin.frontend.version>1.3</plugin.frontend.version>
<!-- common library versions -->
@ -766,6 +766,13 @@
</modules>
</profile>
<profile>
<id>integration</id>
<modules>
<module>zeppelin-integration</module>
</modules>
</profile>
<profile>
<id>r</id>
<modules>
@ -1009,6 +1016,7 @@
<exclude>**/src/fonts/source-code-pro*</exclude>
<exclude>**/src/**/**.test.js</exclude>
<exclude>**/e2e/**/**.spec.js</exclude>
<exclude>package-lock.json</exclude>
<!-- from SQLLine 1.0.2, see ZEPPELIN-2135 -->
<exclude>**/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java</exclude>

9
python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
View file

@ -336,14 +336,19 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
LOGGER.debug("Call completion for: " + buf);
List<InterpreterCompletion> completions = new ArrayList<>();
CompletionResponse response =
ipythonClient.complete(
CompletionRequest.getDefaultInstance().newBuilder().setCode(buf)
.setCursor(cursor).build());
for (int i = 0; i < response.getMatchesCount(); i++) {
completions.add(new InterpreterCompletion(
response.getMatches(i), response.getMatches(i), ""));
String match = response.getMatches(i);
int lastIndexOfDot = match.lastIndexOf(".");
if (lastIndexOfDot != -1) {
match = match.substring(lastIndexOfDot + 1);
}
completions.add(new InterpreterCompletion(match, match, ""));
}
return completions;
}

10
python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
View file

@ -196,9 +196,9 @@ public class IPythonInterpreterTest {
context = getInterpreterContext();
completions = interpreter.completion("sys.std", 7, context);
assertEquals(3, completions.size());
assertEquals("sys.stderr", completions.get(0).getValue());
assertEquals("sys.stdin", completions.get(1).getValue());
assertEquals("sys.stdout", completions.get(2).getValue());
assertEquals("stderr", completions.get(0).getValue());
assertEquals("stdin", completions.get(1).getValue());
assertEquals("stdout", completions.get(2).getValue());
// there's no completion for 'a.' because it is not recognized by compiler for now.
context = getInterpreterContext();
@ -227,14 +227,14 @@ public class IPythonInterpreterTest {
st = "a.co";
completions = interpreter.completion(st, st.length(), context);
assertEquals(1, completions.size());
assertEquals("a.count", completions.get(0).getValue());
assertEquals("count", completions.get(0).getValue());
// cursor is in the middle of code
context = getInterpreterContext();
st = "a.co\b='hello";
completions = interpreter.completion(st, 4, context);
assertEquals(1, completions.size());
assertEquals("a.count", completions.get(0).getValue());
assertEquals("count", completions.get(0).getValue());
// ipython help
context = getInterpreterContext();

1
shell/pom.xml
View file

@ -45,6 +45,7 @@
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>

29
spark/pom.xml
View file

@ -340,19 +340,6 @@
</dependencies>
<build>
<!-- sparkr resources -->
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>interpreter-setting.json</exclude>
</excludes>
</resource>
<resource>
<directory>src/main/sparkr-resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
@ -549,6 +536,22 @@
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-interpreter-setting</id>
<phase>package</phase>
<goals>
<goal>resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/spark</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

39
spark/src/main/resources/interpreter-setting.json
View file

@ -183,5 +183,44 @@
"language": "python",
"editOnDblClick": false
}
},
{
"group": "spark",
"name": "r",
"className": "org.apache.zeppelin.spark.SparkRInterpreter",
"properties": {
"zeppelin.R.knitr": {
"envName": "ZEPPELIN_R_KNITR",
"propertyName": "zeppelin.R.knitr",
"defaultValue": true,
"description": "whether use knitr or not",
"type": "checkbox"
},
"zeppelin.R.cmd": {
"envName": "ZEPPELIN_R_CMD",
"propertyName": "zeppelin.R.cmd",
"defaultValue": "R",
"description": "R repl path",
"type": "string"
},
"zeppelin.R.image.width": {
"envName": "ZEPPELIN_R_IMAGE_WIDTH",
"propertyName": "zeppelin.R.image.width",
"defaultValue": "100%",
"description": "",
"type": "number"
},
"zeppelin.R.render.options": {
"envName": "ZEPPELIN_R_RENDER_OPTIONS",
"propertyName": "zeppelin.R.render.options",
"defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F, fig.retina = 2",
"description": "",
"type": "textarea"
}
},
"editor": {
"language": "r",
"editOnDblClick": false
}
}
]

204
spark/src/main/sparkr-resources/interpreter-setting.json
View file

@ -1,204 +0,0 @@
[
{
"group": "spark",
"name": "spark",
"className": "org.apache.zeppelin.spark.SparkInterpreter",
"defaultInterpreter": true,
"properties": {
"spark.executor.memory": {
"envName": null,
"propertyName": "spark.executor.memory",
"defaultValue": "",
"description": "Executor memory per worker instance. ex) 512m, 32g",
"type": "string"
},
"args": {
"envName": null,
"propertyName": null,
"defaultValue": "",
"description": "spark commandline args",
"type": "string"
},
"zeppelin.spark.useHiveContext": {
"envName": "ZEPPELIN_SPARK_USEHIVECONTEXT",
"propertyName": "zeppelin.spark.useHiveContext",
"defaultValue": true,
"description": "Use HiveContext instead of SQLContext if it is true.",
"type": "checkbox"
},
"spark.app.name": {
"envName": "SPARK_APP_NAME",
"propertyName": "spark.app.name",
"defaultValue": "Zeppelin",
"description": "The name of spark application.",
"type": "string"
},
"zeppelin.spark.printREPLOutput": {
"envName": null,
"propertyName": "zeppelin.spark.printREPLOutput",
"defaultValue": true,
"description": "Print REPL output",
"type": "checkbox"
},
"spark.cores.max": {
"envName": null,
"propertyName": "spark.cores.max",
"defaultValue": "",
"description": "Total number of cores to use. Empty value uses all available core.",
"type": "number"
},
"zeppelin.spark.maxResult": {
"envName": "ZEPPELIN_SPARK_MAXRESULT",
"propertyName": "zeppelin.spark.maxResult",
"defaultValue": "1000",
"description": "Max number of Spark SQL result to display.",
"type": "number"
},
"master": {
"envName": "MASTER",
"propertyName": "spark.master",
"defaultValue": "local[*]",
"description": "Spark master uri. ex) spark://masterhost:7077",
"type": "string"
},
"zeppelin.spark.unSupportedVersionCheck": {
"envName": null,
"propertyName": "zeppelin.spark.enableSupportedVersionCheck",
"defaultValue": true,
"description": "Do not change - developer only setting, not for production use",
"type": "checkbox"
}
},
"editor": {
"language": "scala"
}
},
{
"group": "spark",
"name": "sql",
"className": "org.apache.zeppelin.spark.SparkSqlInterpreter",
"properties": {
"zeppelin.spark.concurrentSQL": {
"envName": "ZEPPELIN_SPARK_CONCURRENTSQL",
"propertyName": "zeppelin.spark.concurrentSQL",
"defaultValue": false,
"description": "Execute multiple SQL concurrently if set true.",
"type": "checkbox"
},
"zeppelin.spark.sql.stacktrace": {
"envName": "ZEPPELIN_SPARK_SQL_STACKTRACE",
"propertyName": "zeppelin.spark.sql.stacktrace",
"defaultValue": false,
"description": "Show full exception stacktrace for SQL queries if set to true.",
"type": "checkbox"
},
"zeppelin.spark.maxResult": {
"envName": "ZEPPELIN_SPARK_MAXRESULT",
"propertyName": "zeppelin.spark.maxResult",
"defaultValue": "1000",
"description": "Max number of Spark SQL result to display.",
"type": "number"
},
"zeppelin.spark.importImplicit": {
"envName": "ZEPPELIN_SPARK_IMPORTIMPLICIT",
"propertyName": "zeppelin.spark.importImplicit",
"defaultValue": true,
"description": "Import implicits, UDF collection, and sql if set true. true by default.",
"type": "checkbox"
}
},
"editor": {
"language": "sql"
}
},
{
"group": "spark",
"name": "dep",
"className": "org.apache.zeppelin.spark.DepInterpreter",
"properties": {
"zeppelin.dep.localrepo": {
"envName": "ZEPPELIN_DEP_LOCALREPO",
"propertyName": null,
"defaultValue": "local-repo",
"description": "local repository for dependency loader",
"type": "string"
},
"zeppelin.dep.additionalRemoteRepository": {
"envName": null,
"propertyName": null,
"defaultValue": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
"description": "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.",
"type": "textarea"
}
},
"editor": {
"language": "scala"
}
},
{
"group": "spark",
"name": "pyspark",
"className": "org.apache.zeppelin.spark.PySparkInterpreter",
"properties": {
"zeppelin.pyspark.python": {
"envName": "PYSPARK_PYTHON",
"propertyName": null,
"defaultValue": "python",
"description": "Python command to run pyspark with",
"type": "string"
}
},
"editor": {
"language": "python"
}
},
{
"group": "spark",
"name": "r",
"className": "org.apache.zeppelin.spark.SparkRInterpreter",
"properties": {
"zeppelin.R.knitr": {
"envName": "ZEPPELIN_R_KNITR",
"propertyName": "zeppelin.R.knitr",
"defaultValue": true,
"description": "whether use knitr or not",
"type": "checkbox"
},
"zeppelin.R.cmd": {
"envName": "ZEPPELIN_R_CMD",
"propertyName": "zeppelin.R.cmd",
"defaultValue": "R",
"description": "R repl path",
"type": "string"
},
"zeppelin.R.image.width": {
"envName": "ZEPPELIN_R_IMAGE_WIDTH",
"propertyName": "zeppelin.R.image.width",
"defaultValue": "100%",
"description": "",
"type": "number"
},
"zeppelin.R.render.options": {
"envName": "ZEPPELIN_R_RENDER_OPTIONS",
"propertyName": "zeppelin.R.render.options",
"defaultValue": "out.format = 'html', comment = NA, echo = FALSE, results = 'asis', message = F, warning = F",
"description": "",
"type": "textarea"
}
},
"editor": {
"language": "r"
}
},
{
"group": "spark",
"name": "ipyspark",
"className": "org.apache.zeppelin.spark.IPySparkInterpreter",
"properties": {},
"editor": {
"language": "python",
"editOnDblClick": false
}
}
]

2
spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
View file

@ -162,7 +162,7 @@ public class IPySparkInterpreterTest {
// completions
List<InterpreterCompletion> completions = iPySparkInterpreter.completion("sc.ran", 6, getInterpreterContext());
assertEquals(1, completions.size());
assertEquals("sc.range", completions.get(0).getValue());
assertEquals("range", completions.get(0).getValue());
// pyspark streaming
context = getInterpreterContext();

220
zeppelin-integration/pom.xml Normal file
View file

@ -0,0 +1,220 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-integration</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Integration Test</name>
<!-- See https://github.com/eirslett/frontend-maven-plugin/issues/229 -->
<prerequisites>
<maven>3.1.0</maven>
</prerequisites>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--test library versions-->
<selenium.java.version>3.8.1</selenium.java.version>
<commons.lang3.version>3.4</commons.lang3.version>
<!--plugin library versions-->
<plugin.failsafe.version>2.16</plugin.failsafe.version>
</properties>
<dependencies>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>${selenium.java.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-zengine</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons.lang3.version}</version>
</dependency>
<dependency>
<groupId>org.rauschig</groupId>
<artifactId>jarchivelib</artifactId>
<version>0.7.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!--test libraries-->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${plugin.failsafe.version}</version>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
<configuration>
<argLine>-Xmx2048m</argLine>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>${plugin.surefire.version}</version>
<configuration combine.children="append">
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
<excludes>
<exclude>${tests.to.exclude}</exclude>
</excludes>
<environmentVariables>
<ZEPPELIN_FORCE_STOP>1</ZEPPELIN_FORCE_STOP>
</environmentVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>start-zeppelin</id>
<phase>pre-integration-test</phase>
<configuration>
<target unless="skipTests">
<exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}"
spawn="true">
<arg value="start"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>stop-zeppelin</id>
<phase>post-integration-test</phase>
<configuration>
<target unless="skipTests">
<exec executable="./zeppelin-daemon.sh" dir="${zeppelin.daemon.package.base}"
spawn="false">
<arg value="stop"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>using-source-tree</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<zeppelin.daemon.package.base>
../bin
</zeppelin.daemon.package.base>
</properties>
</profile>
<profile>
<id>using-packaged-distr</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<zeppelin.daemon.package.base>
../zeppelin-distribution/target/zeppelin-${project.version}/zeppelin-${project.version}/bin
</zeppelin.daemon.package.base>
</properties>
</profile>
</profiles>
</project>

8
zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
View file

@ -103,10 +103,6 @@ abstract public class AbstractZeppelinIT {
});
}
protected static boolean endToEndTestEnabled() {
return null != System.getenv("TEST_SELENIUM");
}
protected void createNewNote() {
clickAndWait(By.xpath("//div[contains(@class, \"col-md-4\")]/div/h5/a[contains(.,'Create new" +
" note')]"));
@ -134,10 +130,6 @@ abstract public class AbstractZeppelinIT {
protected void handleException(String message, Exception e) throws Exception {
LOG.error(message, e);
LogEntries logEntries = driver.manage().logs().get(LogType.BROWSER);
for (LogEntry entry : logEntries) {
LOG.error(new Date(entry.getTimestamp()) + " " + entry.getLevel() + " " + entry.getMessage());
}
File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
LOG.error("ScreenShot::\ndata:image/png;base64," + new String(Base64.encodeBase64(FileUtils.readFileToByteArray(scrFile))));
throw e;

0
zeppelin-server/src/test/java/org/apache/zeppelin/CommandExecutor.java → zeppelin-integration/src/test/java/org/apache/zeppelin/CommandExecutor.java
View file

23
zeppelin-server/src/test/java/org/apache/zeppelin/ProcessData.java → zeppelin-integration/src/test/java/org/apache/zeppelin/ProcessData.java
View file

@ -18,12 +18,16 @@
package org.apache.zeppelin;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.concurrent.TimeUnit;
public class ProcessData {
public enum Types_Of_Data {
OUTPUT,
@ -225,13 +229,16 @@ public class ProcessData {
(System.currentTimeMillis() > unconditionalExitTime));
this.checked_process.destroy();
try {
if ((System.currentTimeMillis() > unconditionalExitTime))
LOG.error("!@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@Unconditional exit occured@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@!\nsome process hag up for more than " + unconditionalExitDelayMinutes + " minutes.");
if ((System.currentTimeMillis() > unconditionalExitTime)) {
LOG.error(
"!@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@Unconditional exit occured@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@!\nsome process hag up for more than "
+ unconditionalExitDelayMinutes + " minutes.");
}
LOG.error("!##################################!");
StringWriter sw = new StringWriter();
new Exception("Exited from buildOutputAndErrorStreamData by timeout").printStackTrace(new PrintWriter(sw)); //Get stack trace
String exceptionAsString = sw.toString();
LOG.error(exceptionAsString);
Exception e = new Exception("Exited from buildOutputAndErrorStreamData by timeout");
e.printStackTrace(new PrintWriter(sw)); //Get stack trace
LOG.error(String.valueOf(e), e);
} catch (Exception ignore) {
LOG.info("Exception in ProcessData while buildOutputAndErrorStreamData ", ignore);
}

104
zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java → zeppelin-integration/src/test/java/org/apache/zeppelin/WebDriverManager.java
View file

@ -17,27 +17,32 @@
package org.apache.zeppelin;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.openqa.selenium.By;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.firefox.FirefoxBinary;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.firefox.FirefoxDriver.SystemProperty;
import org.openqa.selenium.firefox.FirefoxOptions;
import org.openqa.selenium.firefox.FirefoxProfile;
import org.openqa.selenium.firefox.GeckoDriverService;
import org.openqa.selenium.safari.SafariDriver;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.rauschig.jarchivelib.Archiver;
import org.rauschig.jarchivelib.ArchiverFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.fail;
public class WebDriverManager {
@ -45,6 +50,8 @@ public class WebDriverManager {
private static String downLoadsDir = "";
private static String GECKODRIVER_VERSION = "0.19.1";
public static WebDriver getWebDriver() {
WebDriver driver = null;
@ -60,12 +67,9 @@ public class WebDriverManager {
downLoadsDir = FileUtils.getTempDirectory().toString();
String tempPath = downLoadsDir + "/firebug/";
String tempPath = downLoadsDir + "/firefox/";
downloadFireBug(firefoxVersion, tempPath);
final String firebugPath = tempPath + "firebug.xpi";
final String firepathPath = tempPath + "firepath.xpi";
downloadGeekoDriver(firefoxVersion, tempPath);
FirefoxProfile profile = new FirefoxProfile();
profile.setPreference("browser.download.folderList", 2);
@ -78,14 +82,17 @@ public class WebDriverManager {
profile.setPreference("app.update.enabled", false);
profile.setPreference("dom.max_script_run_time", 0);
profile.setPreference("dom.max_chrome_script_run_time", 0);
profile.setPreference("browser.helperApps.neverAsk.saveToDisk", "application/x-ustar,application/octet-stream,application/zip,text/csv,text/plain");
profile.setPreference("browser.helperApps.neverAsk.saveToDisk",
"application/x-ustar,application/octet-stream,application/zip,text/csv,text/plain");
profile.setPreference("network.proxy.type", 0);
// Commenting out installing extensions. See ZEPPELIN-2962.
// profile.addExtension(new File(firebugPath));
// profile.addExtension(new File(firepathPath));
System.setProperty(GeckoDriverService.GECKO_DRIVER_EXE_PROPERTY, tempPath + "geckodriver");
System.setProperty(SystemProperty.DRIVER_USE_MARIONETTE, "false");
driver = new FirefoxDriver(ffox, profile);
FirefoxOptions firefoxOptions = new FirefoxOptions();
firefoxOptions.setBinary(ffox);
firefoxOptions.setProfile(profile);
driver = new FirefoxDriver(firefoxOptions);
} catch (Exception e) {
LOG.error("Exception in WebDriverManager while FireFox Driver ", e);
}
@ -146,37 +153,48 @@ public class WebDriverManager {
return driver;
}
private static void downloadFireBug(int firefoxVersion, String tempPath) {
String firebugUrlString = null;
if (firefoxVersion < 23)
firebugUrlString = "http://getfirebug.com/releases/firebug/1.11/firebug-1.11.4.xpi";
else if (firefoxVersion >= 23 && firefoxVersion < 30)
firebugUrlString = "http://getfirebug.com/releases/firebug/1.12/firebug-1.12.8.xpi";
else if (firefoxVersion >= 30 && firefoxVersion < 33)
firebugUrlString = "http://getfirebug.com/releases/firebug/2.0/firebug-2.0.7.xpi";
else if (firefoxVersion >= 33)
firebugUrlString = "http://getfirebug.com/releases/firebug/2.0/firebug-2.0.17.xpi";
public static void downloadGeekoDriver(int firefoxVersion, String tempPath) {
String geekoDriverUrlString =
"https://github.com/mozilla/geckodriver/releases/download/v" + GECKODRIVER_VERSION
+ "/geckodriver-v" + GECKODRIVER_VERSION + "-";
LOG.info("firebug version: " + firefoxVersion + ", will be downloaded to " + tempPath);
LOG.info("Geeko version: " + firefoxVersion + ", will be downloaded to " + tempPath);
try {
File firebugFile = new File(tempPath + "firebug.xpi");
URL firebugUrl = new URL(firebugUrlString);
if (!firebugFile.exists()) {
FileUtils.copyURLToFile(firebugUrl, firebugFile);
if (SystemUtils.IS_OS_WINDOWS) {
if (System.getProperty("sun.arch.data.model").equals("64")) {
geekoDriverUrlString += "win64.zip";
} else {
geekoDriverUrlString += "win32.zip";
}
} else if (SystemUtils.IS_OS_LINUX) {
if (System.getProperty("sun.arch.data.model").equals("64")) {
geekoDriverUrlString += "linux64.tar.gz";
} else {
geekoDriverUrlString += "linux32.tar.gz";
}
} else if (SystemUtils.IS_OS_MAC_OSX) {
geekoDriverUrlString += "macos.tar.gz";
}
File firepathFile = new File(tempPath + "firepath.xpi");
URL firepathUrl = new URL("https://addons.cdn.mozilla.net/user-media/addons/11900/firepath-0.9.7.1-fx.xpi");
if (!firepathFile.exists()) {
FileUtils.copyURLToFile(firepathUrl, firepathFile);
File geekoDriver = new File(tempPath + "geckodriver");
File geekoDriverZip = new File(tempPath + "geckodriver.tar");
File geekoDriverDir = new File(tempPath);
URL geekoDriverUrl = new URL(geekoDriverUrlString);
if (!geekoDriver.exists()) {
FileUtils.copyURLToFile(geekoDriverUrl, geekoDriverZip);
if (SystemUtils.IS_OS_WINDOWS) {
Archiver archiver = ArchiverFactory.createArchiver("zip");
archiver.extract(geekoDriverZip, geekoDriverDir);
} else {
Archiver archiver = ArchiverFactory.createArchiver("tar", "gz");
archiver.extract(geekoDriverZip, geekoDriverDir);
}
}
} catch (IOException e) {
LOG.error("Download of firebug version: " + firefoxVersion + ", falied in path " + tempPath);
LOG.error("Download of Geeko version: " + firefoxVersion + ", falied in path " + tempPath);
}
LOG.info("Download of firebug version: " + firefoxVersion + ", successful");
LOG.info("Download of Geeko version: " + firefoxVersion + ", successful");
}
public static int getFirefoxVersion() {
@ -185,8 +203,10 @@ public class WebDriverManager {
if (System.getProperty("os.name").startsWith("Mac OS")) {
firefoxVersionCmd = "/Applications/Firefox.app/Contents/MacOS/" + firefoxVersionCmd;
}
String versionString = (String) CommandExecutor.executeCommandLocalHost(firefoxVersionCmd, false, ProcessData.Types_Of_Data.OUTPUT);
return Integer.valueOf(versionString.replaceAll("Mozilla Firefox", "").trim().substring(0, 2));
String versionString = (String) CommandExecutor
.executeCommandLocalHost(firefoxVersionCmd, false, ProcessData.Types_Of_Data.OUTPUT);
return Integer
.valueOf(versionString.replaceAll("Mozilla Firefox", "").trim().substring(0, 2));
} catch (Exception e) {
LOG.error("Exception in WebDriverManager while getWebDriver ", e);
return -1;

60
zeppelin-integration/src/test/java/org/apache/zeppelin/ZeppelinITUtils.java Normal file
View file

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.openqa.selenium.WebDriver;
import java.util.concurrent.TimeUnit;
public class ZeppelinITUtils {
public final static Logger LOG = LoggerFactory.getLogger(ZeppelinITUtils.class);
public static void sleep(long millis, boolean logOutput) {
if (logOutput) {
LOG.info("Starting sleeping for " + (millis / 1000) + " seconds...");
LOG.info("Caller: " + Thread.currentThread().getStackTrace()[2]);
}
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
LOG.error("Exception in WebDriverManager while getWebDriver ", e);
}
if (logOutput) {
LOG.info("Finished.");
}
}
public static void restartZeppelin() {
CommandExecutor.executeCommandLocalHost("../bin/zeppelin-daemon.sh restart",
false, ProcessData.Types_Of_Data.OUTPUT);
//wait for server to start.
sleep(5000, false);
}
public static void turnOffImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(0, TimeUnit.SECONDS);
}
public static void turnOnImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(AbstractZeppelinIT.MAX_IMPLICIT_WAIT,
TimeUnit.SECONDS);
}
}

19
zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
View file

@ -78,10 +78,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
@BeforeClass
public static void startUp() {
if (!endToEndTestEnabled()) {
return;
}
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
@ -101,9 +97,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
@AfterClass
public static void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
try {
if (!StringUtils.isBlank(shiroPath)) {
File file = new File(shiroPath);
@ -133,9 +126,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
}
private void testShowNotebookListOnNavbar() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
MAX_BROWSER_TIMEOUT_SEC).click();
@ -167,9 +157,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
// @Test
public void testSimpleAuthentication() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
AuthenticationIT authenticationIT = new AuthenticationIT();
authenticationIT.authenticationUser("admin", "password1");
@ -186,9 +173,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
@Test
public void testAnyOfRoles() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
AuthenticationIT authenticationIT = new AuthenticationIT();
authenticationIT.authenticationUser("admin", "password1");
@ -242,9 +226,6 @@ public class AuthenticationIT extends AbstractZeppelinIT {
@Test
public void testGroupPermission() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
AuthenticationIT authenticationIT = new AuthenticationIT();
authenticationIT.authenticationUser("finance1", "finance1");

11
zeppelin-server/src/test/java/org/apache/zeppelin/integration/InterpreterIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterIT.java
View file

@ -39,25 +39,16 @@ public class InterpreterIT extends AbstractZeppelinIT {
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
}
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
driver.quit();
}
@Test
public void testShowDescriptionOnInterpreterCreate() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// navigate to interpreter page
WebElement settingButton = driver.findElement(By.xpath("//button[@class='nav-btn dropdown-toggle ng-scope']"));
@ -79,4 +70,4 @@ public class InterpreterIT extends AbstractZeppelinIT {
handleException("Exception in InterpreterIT while testShowDescriptionOnInterpreterCreate ", e);
}
}
}
}

15
zeppelin-server/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java
View file

@ -78,9 +78,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
@BeforeClass
public static void startUp() {
if (!endToEndTestEnabled()) {
return;
}
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
@ -105,9 +102,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
@AfterClass
public static void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
try {
if (!StringUtils.isBlank(shiroPath)) {
File shiroFile = new File(shiroPath);
@ -174,9 +168,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
@Test
public void testGloballyAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
//step 1: (admin) login, set 'globally in shared' mode of python interpreter, logout
InterpreterModeActionsIT interpreterModeActionsIT = new InterpreterModeActionsIT();
@ -326,9 +317,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
@Test
public void testPerUserScopedAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
//step 1: (admin) login, set 'Per user in scoped' mode of python interpreter, logout
InterpreterModeActionsIT interpreterModeActionsIT = new InterpreterModeActionsIT();
@ -611,9 +599,6 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
@Test
public void testPerUserIsolatedAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
//step 1: (admin) login, set 'Per user in isolated' mode of python interpreter, logout
InterpreterModeActionsIT interpreterModeActionsIT = new InterpreterModeActionsIT();

66
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -45,26 +45,16 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
}
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
driver.quit();
}
@Test
public void testCreateNewButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
Actions action = new Actions(driver);
@ -133,9 +123,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testRemoveButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -170,9 +157,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testMoveUpAndDown() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -223,9 +207,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testDisableParagraphRunButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -257,9 +238,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testRunOnSelectionChange() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
String xpathToRunOnSelectionChangeCheckbox = getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]";
String xpathToDropdownMenu = getParagraphXPath(1) + "//select";
@ -313,9 +291,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testClearOutputButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -345,9 +320,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testWidth() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
waitForParagraph(1, "READY");
@ -372,9 +344,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testFontSize() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
waitForParagraph(1, "READY");
@ -400,9 +369,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testTitleButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -471,9 +437,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testShowAndHideLineNumbers() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -519,9 +482,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testEditOnDoubleClick() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
Actions action = new Actions(driver);
@ -568,9 +528,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testSingleDynamicFormTextInput() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -604,9 +561,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testSingleDynamicFormSelectForm() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -644,9 +598,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testSingleDynamicFormCheckboxForm() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -687,9 +638,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testMultipleDynamicFormsSameType() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -728,9 +676,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testNoteDynamicFormTextInput() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -769,9 +714,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testNoteDynamicFormSelect() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -817,9 +759,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testDynamicNoteFormCheckbox() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -864,9 +803,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
@Test
public void testWithNoteAndParagraphDynamicFormTextInput() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -885,4 +821,4 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
handleException("Exception in ParagraphActionsIT while testWithNoteAndParagraphDynamicFormTextInput ", e);
}
}
}
}

15
zeppelin-server/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java
View file

@ -70,9 +70,6 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
@BeforeClass
public static void startUp() {
if (!endToEndTestEnabled()) {
return;
}
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
@ -91,9 +88,6 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
@AfterClass
public static void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
try {
if (!StringUtils.isBlank(shiroPath)) {
File file = new File(shiroPath);
@ -118,9 +112,6 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
@Test
public void testSimpleAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// step 1 : (admin) create a new note, run a paragraph and turn on personalized mode
AuthenticationIT authenticationIT = new AuthenticationIT();
@ -197,9 +188,6 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
@Test
public void testGraphAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// step 1 : (admin) create a new note, run a paragraph, change active graph to 'Bar chart', turn on personalized mode
AuthenticationIT authenticationIT = new AuthenticationIT();
@ -273,9 +261,6 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
@Test
public void testDynamicFormAction() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// step 1 : (admin) login, create a new note, run a paragraph with data of spark tutorial, logout.
AuthenticationIT authenticationIT = new AuthenticationIT();

18
zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
View file

@ -42,9 +42,6 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
createNewNote();
waitForParagraph(1, "READY");
@ -52,18 +49,12 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
deleteTestNotebook(driver);
driver.quit();
}
@Test
public void testSpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1, "sc.version");
runParagraph(1);
@ -116,9 +107,6 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
@Test
public void testPySpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1, "%pyspark\\n" +
"for x in range(0, 3):\\n" +
@ -167,9 +155,6 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
@Test
public void testSqlSpark() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
setTextOfParagraph(1,"%sql\\n" +
"select * from bank limit 1");
@ -201,9 +186,6 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
@Test
public void testDep() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// restart spark interpreter before running %dep
clickAndWait(By.xpath("//span[@uib-tooltip='Interpreter binding']"));

17
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java → zeppelin-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
View file

@ -57,26 +57,16 @@ public class ZeppelinIT extends AbstractZeppelinIT {
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
}
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
driver.quit();
}
@Test
public void testAngularDisplay() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
@ -209,9 +199,6 @@ public class ZeppelinIT extends AbstractZeppelinIT {
@Test
public void testSparkInterpreterDependencyLoading() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// navigate to interpreter page
WebElement settingButton = driver.findElement(By.xpath("//button[@class='nav-btn dropdown-toggle ng-scope']"));
@ -279,10 +266,6 @@ public class ZeppelinIT extends AbstractZeppelinIT {
@Test
public void testAngularRunParagraph() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();

46
zeppelin-integration/src/test/resources/log4j.properties Normal file
View file

@ -0,0 +1,46 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Direct log messages to stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
#log4j.appender.stdout.layout.ConversionPattern=
#%5p [%t] (%F:%L) - %m%n
#%-4r [%t] %-5p %c %x - %m%n
#
# Root logger option
log4j.rootLogger=INFO, stdout
#mute some noisy guys
log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
log4j.logger.DataNucleus.MetaData=ERROR
log4j.logger.DataNucleus.Datastore=ERROR
# Log all JDBC parameters
log4j.logger.org.hibernate.type=ALL
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
log4j.logger.org.apache.zeppelin.spark=DEBUG

99
zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -45,11 +45,29 @@ public class ZeppelinConfiguration extends XMLConfiguration {
"https://s3.amazonaws.com/helium-package/helium.json";
private static ZeppelinConfiguration conf;
private Map<String, String> properties = new HashMap<>();
public ZeppelinConfiguration(URL url) throws ConfigurationException {
setDelimiterParsingDisabled(true);
load(url);
initProperties();
}
private void initProperties() {
List<ConfigurationNode> nodes = getRootNode().getChildren();
if (nodes == null || nodes.isEmpty()) {
return;
}
for (ConfigurationNode p : nodes) {
String name = (String) p.getChildren("name").get(0).getValue();
String value = (String) p.getChildren("value").get(0).getValue();
if (!StringUtils.isEmpty(name)) {
properties.put(name, value);
}
}
}
public ZeppelinConfiguration() {
ConfVars[] vars = ConfVars.values();
for (ConfVars v : vars) {
@ -122,71 +140,41 @@ public class ZeppelinConfiguration extends XMLConfiguration {
private String getStringValue(String name, String d) {
List<ConfigurationNode> properties = getRootNode().getChildren();
if (properties == null || properties.isEmpty()) {
return d;
}
for (ConfigurationNode p : properties) {
if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
&& name.equals(p.getChildren("name").get(0).getValue())) {
return (String) p.getChildren("value").get(0).getValue();
}
String value = this.properties.get(name);
if (value != null) {
return value;
}
return d;
}
private int getIntValue(String name, int d) {
List<ConfigurationNode> properties = getRootNode().getChildren();
if (properties == null || properties.isEmpty()) {
return d;
}
for (ConfigurationNode p : properties) {
if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
&& name.equals(p.getChildren("name").get(0).getValue())) {
return Integer.parseInt((String) p.getChildren("value").get(0).getValue());
}
String value = this.properties.get(name);
if (value != null) {
return Integer.parseInt(value);
}
return d;
}
private long getLongValue(String name, long d) {
List<ConfigurationNode> properties = getRootNode().getChildren();
if (properties == null || properties.isEmpty()) {
return d;
}
for (ConfigurationNode p : properties) {
if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
&& name.equals(p.getChildren("name").get(0).getValue())) {
return Long.parseLong((String) p.getChildren("value").get(0).getValue());
}
String value = this.properties.get(name);
if (value != null) {
return Long.parseLong(value);
}
return d;
}
private float getFloatValue(String name, float d) {
List<ConfigurationNode> properties = getRootNode().getChildren();
if (properties == null || properties.isEmpty()) {
return d;
}
for (ConfigurationNode p : properties) {
if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
&& name.equals(p.getChildren("name").get(0).getValue())) {
return Float.parseFloat((String) p.getChildren("value").get(0).getValue());
}
String value = this.properties.get(name);
if (value != null) {
return Float.parseFloat(value);
}
return d;
}
private boolean getBooleanValue(String name, boolean d) {
List<ConfigurationNode> properties = getRootNode().getChildren();
if (properties == null || properties.isEmpty()) {
return d;
}
for (ConfigurationNode p : properties) {
if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
&& name.equals(p.getChildren("name").get(0).getValue())) {
return Boolean.parseBoolean((String) p.getChildren("value").get(0).getValue());
}
String value = this.properties.get(name);
if (value != null) {
return Boolean.parseBoolean(value);
}
return d;
}
@ -265,6 +253,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getBooleanValue(propertyName, defaultValue);
}
public String getZeppelinHome() {
return getString(ConfVars.ZEPPELIN_HOME);
}
public boolean useSsl() {
return getBoolean(ConfVars.ZEPPELIN_SSL);
}
@ -429,7 +421,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getInterpreterSettingPath() {
return getRelativeDir(String.format("%s/interpreter.json", getConfDir()));
return getConfigFSDir() + "/interpreter.json";
}
public String getHeliumConfPath() {
@ -453,7 +445,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getNotebookAuthorizationPath() {
return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
return getConfigFSDir() + "/notebook-authorization.json";
}
public Boolean credentialsPersist() {
@ -521,6 +513,16 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getRelativeDir(ConfVars.ZEPPELIN_CONF_DIR);
}
public String getConfigFSDir() {
String fsConfigDir = getString(ConfVars.ZEPPELIN_CONFIG_FS_DIR);
if (StringUtils.isBlank(fsConfigDir)) {
LOG.warn(ConfVars.ZEPPELIN_CONFIG_FS_DIR.varName + " is not specified, fall back to local " +
"conf directory " + ConfVars.ZEPPELIN_CONF_DIR.varName);
return "file://" + getConfDir();
}
return fsConfigDir;
}
public List<String> getAllowedOrigins()
{
if (getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).isEmpty()) {
@ -721,6 +723,9 @@ public class ZeppelinConfiguration extends XMLConfiguration {
// Decide when new note is created, interpreter settings will be binded automatically or not.
ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
ZEPPELIN_CONFIG_FS_DIR("zeppelin.config.fs.dir", ""),
ZEPPELIN_CONFIG_STORAGE_CLASS("zeppelin.config.storage.class",
"org.apache.zeppelin.storage.FileSystemConfigStorage"),
ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url",

19
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java
View file

@ -20,14 +20,15 @@ package org.apache.zeppelin.interpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URISyntaxException;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
/**
* InterpreterOutput is OutputStream that supposed to print content on notebook
@ -36,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class InterpreterOutput extends OutputStream {
Logger logger = LoggerFactory.getLogger(InterpreterOutput.class);
private final int NEW_LINE_CHAR = '\n';
private final int LINE_FEED_CHAR = '\r';
private List<InterpreterResultMessageOutput> resultMessageOutputs = new LinkedList<>();
private InterpreterResultMessageOutput currentOut;
@ -47,6 +49,7 @@ public class InterpreterOutput extends OutputStream {
private final InterpreterOutputChangeListener changeListener;
private int size = 0;
private int lastCRIndex = -1;
// change static var to set interpreter output limit
// limit will be applied to all InterpreterOutput object.
@ -84,6 +87,7 @@ public class InterpreterOutput extends OutputStream {
buffer.reset();
size = 0;
lastCRIndex = -1;
if (currentOut != null) {
currentOut.flush();
@ -146,6 +150,7 @@ public class InterpreterOutput extends OutputStream {
public void clear() {
size = 0;
lastCRIndex = -1;
truncated = false;
buffer.reset();
@ -204,6 +209,14 @@ public class InterpreterOutput extends OutputStream {
}
}
if (b == LINE_FEED_CHAR) {
if (lastCRIndex == -1) {
lastCRIndex = size;
}
// reset size to index of last carriage return
size = lastCRIndex;
}
if (startOfTheNewLine) {
if (b == '%') {
startOfTheNewLine = false;

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
View file

@ -26,7 +26,7 @@ public interface InterpreterClient {
String getInterpreterSettingName();
void start(String userName, Boolean isUserImpersonate);
void start(String userName);
void stop();

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java
View file

@ -30,6 +30,7 @@ public class InterpreterLaunchContext {
private Properties properties;
private InterpreterOption option;
private InterpreterRunner runner;
private String userName;
private String interpreterGroupId;
private String interpreterSettingId;
private String interpreterSettingGroup;
@ -38,6 +39,7 @@ public class InterpreterLaunchContext {
public InterpreterLaunchContext(Properties properties,
InterpreterOption option,
InterpreterRunner runner,
String userName,
String interpreterGroupId,
String interpreterSettingId,
String interpreterSettingGroup,
@ -45,6 +47,7 @@ public class InterpreterLaunchContext {
this.properties = properties;
this.option = option;
this.runner = runner;
this.userName = userName;
this.interpreterGroupId = interpreterGroupId;
this.interpreterSettingId = interpreterSettingId;
this.interpreterSettingGroup = interpreterSettingGroup;
@ -78,4 +81,8 @@ public class InterpreterLaunchContext {
public String getInterpreterSettingName() {
return interpreterSettingName;
}
public String getUserName() {
return userName;
}
}

2
zeppelin-jupyter/pom.xml
View file

@ -35,7 +35,7 @@
<description>Jupyter support for Apache Zeppelin</description>
<properties>
<zeppelin.version>0.8.0-SNAPSHOT</zeppelin.version>
<zeppelin.version>${project.version}</zeppelin.version>
</properties>
<dependencies>

7
zeppelin-server/pom.xml
View file

@ -37,7 +37,6 @@
<!--library versions-->
<commons.httpclient.version>4.3.6</commons.httpclient.version>
<jersey.version>2.22.2</jersey.version>
<hadoop-common.version>2.6.0</hadoop-common.version>
<quartz.scheduler.version>2.2.1</quartz.scheduler.version>
<jersey.servlet.version>1.13</jersey.servlet.version>
<javax.ws.rsapi.version>2.0.1</javax.ws.rsapi.version>
@ -214,6 +213,12 @@
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
<version>4.41.2</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>

50
zeppelin-server/src/test/scala/org/apache/zeppelin/WelcomePageSuite.scala → zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/JWTAuthenticationToken.java
View file

@ -14,24 +14,46 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
package org.apache.zeppelin
import org.apache.shiro.authc.AuthenticationToken;
import org.openqa.selenium.WebDriver
import org.scalatest.concurrent.Eventually._
import org.scalatest.time._
import org.scalatest.selenium.WebBrowser
import org.scalatest.{DoNotDiscover, FunSuite}
import AbstractFunctionalSuite.SERVER_ADDRESS
/**
* Created for org.apache.zeppelin.server
*/
public class JWTAuthenticationToken implements AuthenticationToken {
@DoNotDiscover
class WelcomePageSuite(implicit driver: WebDriver) extends FunSuite with WebBrowser {
private Object userId;
private String token;
test("Welcome sign is correct") {
eventually (timeout(Span(180, Seconds))) {
go to SERVER_ADDRESS
assert(find("welcome").isDefined)
}
public JWTAuthenticationToken(Object userId, String token) {
this.userId = userId;
this.token = token;
}
@Override
public Object getPrincipal() {
return getUserId();
}
@Override
public Object getCredentials() {
return getToken();
}
public Object getUserId() {
return userId;
}
public void setUserId(long userId) {
this.userId = userId;
}
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
}

71
zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxAuthenticationFilter.java Normal file
View file

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
import org.apache.shiro.web.filter.authc.FormAuthenticationFilter;
import org.apache.shiro.web.servlet.ShiroHttpServletRequest;
import org.apache.zeppelin.utils.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
/**
* Created for org.apache.zeppelin.server
*/
public class KnoxAuthenticationFilter extends FormAuthenticationFilter {
private static final Logger LOGGER = LoggerFactory.getLogger(KnoxAuthenticationFilter.class);
protected boolean isAccessAllowed(ServletRequest request,
ServletResponse response, Object mappedValue) {
//Check with existing shiro authentication logic
//https://github.com/apache/shiro/blob/shiro-root-1.3.2/web/src/main/java/org/apache/shiro/
// web/filter/authc/AuthenticatingFilter.java#L123-L124
Boolean accessAllowed = super.isAccessAllowed(request, response, mappedValue) ||
!isLoginRequest(request, response) && isPermissive(mappedValue);
if (accessAllowed) {
accessAllowed = false;
KnoxJwtRealm knoxJwtRealm = null;
for (Object realm : SecurityUtils.getRealmsList()) {
if (realm instanceof KnoxJwtRealm) {
knoxJwtRealm = (KnoxJwtRealm) realm;
break;
}
}
if (knoxJwtRealm != null) {
for (Cookie cookie : ((ShiroHttpServletRequest) request).getCookies()) {
if (cookie.getName().equals(knoxJwtRealm.getCookieName())) {
if (knoxJwtRealm.validateToken(cookie.getValue())) {
accessAllowed = true;
}
break;
}
}
} else {
LOGGER.error("Looks like this filter is enabled without enabling KnoxJwtRealm, please refer"
+ " to https://zeppelin.apache.org/docs/latest/security/shiroauthentication.html"
+ "#knox-sso");
}
}
return accessAllowed;
}
}

289
zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/KnoxJwtRealm.java Normal file
View file

@ -0,0 +1,289 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
import com.nimbusds.jose.JWSObject;
import com.nimbusds.jose.JWSVerifier;
import com.nimbusds.jose.crypto.RSASSAVerifier;
import com.nimbusds.jwt.SignedJWT;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.PublicKey;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.interfaces.RSAPublicKey;
import java.text.ParseException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.servlet.ServletException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Groups;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAccount;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Created for org.apache.zeppelin.server
*/
public class KnoxJwtRealm extends AuthorizingRealm {
private static final Logger LOGGER = LoggerFactory.getLogger(KnoxJwtRealm.class);
private String providerUrl;
private String redirectParam;
private String cookieName;
private String publicKeyPath;
private String login;
private String logout;
private String principalMapping;
private String groupPrincipalMapping;
private SimplePrincipalMapper mapper = new SimplePrincipalMapper();
/**
* Configuration object needed by for hadoop classes
*/
private Configuration hadoopConfig;
/**
* Hadoop Groups implementation.
*/
private Groups hadoopGroups;
@Override
protected void onInit() {
super.onInit();
if (principalMapping != null && !principalMapping.isEmpty()
|| groupPrincipalMapping != null && !groupPrincipalMapping.isEmpty()) {
try {
mapper.loadMappingTable(principalMapping, groupPrincipalMapping);
} catch (PrincipalMappingException e) {
LOGGER.error("PrincipalMappingException in onInit", e);
}
}
try {
hadoopConfig = new Configuration();
hadoopGroups = new Groups(hadoopConfig);
} catch (final Exception e) {
LOGGER.error("Exception in onInit", e);
}
}
@Override
public boolean supports(AuthenticationToken token) {
return token != null && token instanceof JWTAuthenticationToken;
}
@Override
protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) {
JWTAuthenticationToken upToken = (JWTAuthenticationToken) token;
if (validateToken(upToken.getToken())) {
try {
SimpleAccount account = new SimpleAccount(getName(upToken), upToken.getToken(), getName());
account.addRole(mapGroupPrincipals(getName(upToken)));
return account;
} catch (ParseException e) {
LOGGER.error("ParseException in doGetAuthenticationInfo", e);
}
}
return null;
}
private String getName(JWTAuthenticationToken upToken) throws ParseException {
SignedJWT signed = SignedJWT.parse(upToken.getToken());
String userName = signed.getJWTClaimsSet().getSubject();
return userName;
}
protected boolean validateToken(String token) {
try {
SignedJWT signed = SignedJWT.parse(token);
return validateSignature(signed);
} catch (ParseException ex) {
LOGGER.info("ParseException in validateToken", ex);
return false;
}
}
public static RSAPublicKey parseRSAPublicKey(String pem)
throws IOException, ServletException {
String PEM_HEADER = "-----BEGIN CERTIFICATE-----\n";
String PEM_FOOTER = "\n-----END CERTIFICATE-----";
String fullPem = PEM_HEADER + pem + PEM_FOOTER;
PublicKey key = null;
try {
CertificateFactory fact = CertificateFactory.getInstance("X.509");
ByteArrayInputStream is = new ByteArrayInputStream(
FileUtils.readFileToString(new File(pem)).getBytes("UTF8"));
X509Certificate cer = (X509Certificate) fact.generateCertificate(is);
key = cer.getPublicKey();
} catch (CertificateException ce) {
String message = null;
if (pem.startsWith(PEM_HEADER)) {
message = "CertificateException - be sure not to include PEM header "
+ "and footer in the PEM configuration element.";
} else {
message = "CertificateException - PEM may be corrupt";
}
throw new ServletException(message, ce);
} catch (UnsupportedEncodingException uee) {
throw new ServletException(uee);
} catch (IOException e) {
throw new IOException(e);
}
return (RSAPublicKey) key;
}
protected boolean validateSignature(SignedJWT jwtToken) {
boolean valid = false;
if (JWSObject.State.SIGNED == jwtToken.getState()) {
if (jwtToken.getSignature() != null) {
try {
RSAPublicKey publicKey = parseRSAPublicKey(publicKeyPath);
JWSVerifier verifier = new RSASSAVerifier(publicKey);
if (verifier != null && jwtToken.verify(verifier)) {
valid = true;
}
} catch (Exception e) {
LOGGER.info("Exception in validateSignature", e);
}
}
}
return valid;
}
@Override
protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
Set<String> roles = mapGroupPrincipals(principals.toString());
return new SimpleAuthorizationInfo(roles);
}
/**
* Query the Hadoop implementation of {@link Groups} to retrieve groups for
* provided user.
*/
public Set<String> mapGroupPrincipals(final String mappedPrincipalName) {
/* return the groups as seen by Hadoop */
Set<String> groups = null;
try {
hadoopGroups.refresh();
final List<String> groupList = hadoopGroups
.getGroups(mappedPrincipalName);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(String.format("group found %s, %s",
mappedPrincipalName, groupList.toString()));
}
groups = new HashSet<>(groupList);
} catch (final IOException e) {
if (e.toString().contains("No groups found for user")) {
/* no groups found move on */
LOGGER.info(String.format("No groups found for user %s", mappedPrincipalName));
} else {
/* Log the error and return empty group */
LOGGER.info(String.format("errorGettingUserGroups for %s", mappedPrincipalName));
}
groups = new HashSet();
}
return groups;
}
public String getProviderUrl() {
return providerUrl;
}
public void setProviderUrl(String providerUrl) {
this.providerUrl = providerUrl;
}
public String getRedirectParam() {
return redirectParam;
}
public void setRedirectParam(String redirectParam) {
this.redirectParam = redirectParam;
}
public String getCookieName() {
return cookieName;
}
public void setCookieName(String cookieName) {
this.cookieName = cookieName;
}
public String getPublicKeyPath() {
return publicKeyPath;
}
public void setPublicKeyPath(String publicKeyPath) {
this.publicKeyPath = publicKeyPath;
}
public String getLogin() {
return login;
}
public void setLogin(String login) {
this.login = login;
}
public String getLogout() {
return logout;
}
public void setLogout(String logout) {
this.logout = logout;
}
public String getPrincipalMapping() {
return principalMapping;
}
public void setPrincipalMapping(String principalMapping) {
this.principalMapping = principalMapping;
}
public String getGroupPrincipalMapping() {
return groupPrincipalMapping;
}
public void setGroupPrincipalMapping(String groupPrincipalMapping) {
this.groupPrincipalMapping = groupPrincipalMapping;
}
}

51
zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/PrincipalMapper.java Normal file
View file

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
/***
*
*/
public interface PrincipalMapper {
/**
* Load the internal principal mapping table from the provided
* string value which conforms to the following semicolon delimited format:
* actual[,another-actual]=mapped;...
* @param principalMapping
*/
public abstract void loadMappingTable(String principalMapping, String groupMapping)
throws PrincipalMappingException;
/**
* Acquire a mapped principal name from the mapping table
* as appropriate. Otherwise, the provided principalName
* will be used.
* @param principalName
* @return principal name to be used in the assertion
*/
public abstract String mapUserPrincipal(String principalName);
/**
* Acquire array of group principal names from the mapping table
* as appropriate. Otherwise, return null.
* @param principalName
* @return group principal names to be used in the assertion
*/
public abstract String[] mapGroupPrincipal(String principalName);
}

34
zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/PrincipalMappingException.java Normal file
View file

@ -0,0 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
/***
* {@link System}
*/
public class PrincipalMappingException extends Exception {
public PrincipalMappingException(String message) {
super(message);
}
public PrincipalMappingException(String message, Exception e) {
super(message, e);
}
}

126
zeppelin-server/src/main/java/org/apache/zeppelin/realm/jwt/SimplePrincipalMapper.java Normal file
View file

@ -0,0 +1,126 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.realm.jwt;
import java.util.Arrays;
import java.util.HashMap;
import java.util.StringTokenizer;
/***
*
*/
public class SimplePrincipalMapper implements PrincipalMapper {
public HashMap<String, String[]> principalMappings = null;
public HashMap<String, String[]> groupMappings = null;
public SimplePrincipalMapper() {
}
/* (non-Javadoc)
* @see org.apache.hadoop.gateway.filter.PrincipalMapper#loadMappingTable(java.lang.String)
*/
@Override
public void loadMappingTable(String principalMapping, String groupMapping)
throws PrincipalMappingException {
if (principalMapping != null) {
principalMappings = parseMapping(principalMapping);
groupMappings = parseMapping(groupMapping);
}
}
private HashMap<String, String[]> parseMapping(String mappings)
throws PrincipalMappingException {
if (mappings == null) {
return null;
}
HashMap<String, String[]> table = new HashMap<>();
try {
StringTokenizer t = new StringTokenizer(mappings, ";");
if (t.hasMoreTokens()) {
do {
String mapping = t.nextToken();
String principals = mapping.substring(0, mapping.indexOf('='));
String value = mapping.substring(mapping.indexOf('=') + 1);
String[] v = value.split(",");
String[] p = principals.split(",");
for (int i = 0; i < p.length; i++) {
table.put(p[i], v);
}
} while (t.hasMoreTokens());
}
return table;
} catch (Exception e) {
// do not leave table in an unknown state - clear it instead
// no principal mapping will occur
table.clear();
throw new PrincipalMappingException(
"Unable to load mappings from provided string: " + mappings
+ " - no principal mapping will be provided.", e);
}
}
/* (non-Javadoc)
* @see org.apache.hadoop.gateway.filter.PrincipalMapper#mapPrincipal(java.lang.String)
*/
@Override
public String mapUserPrincipal(String principalName) {
String[] p = null;
if (principalMappings != null) {
p = principalMappings.get(principalName);
}
if (p == null) {
return principalName;
}
return p[0];
}
/* (non-Javadoc)
* @see org.apache.hadoop.gateway.filter.PrincipalMapper#mapPrincipal(java.lang.String)
*/
@Override
public String[] mapGroupPrincipal(String principalName) {
String[] groups = null;
String[] wildCardGroups = null;
if (groupMappings != null) {
groups = groupMappings.get(principalName);
wildCardGroups = groupMappings.get("*");
if (groups != null && wildCardGroups != null) {
groups = concat(groups, wildCardGroups);
} else if (wildCardGroups != null) {
return wildCardGroups;
}
}
return groups;
}
/**
* @param groups
* @param wildCardGroups
* @return
*/
public static <T> T[] concat(T[] groups, T[] wildCardGroups) {
T[] result = Arrays.copyOf(groups, groups.length + wildCardGroups.length);
System.arraycopy(wildCardGroups, 0, result, groups.length, wildCardGroups.length);
return result;
}
}

193
zeppelin-server/src/main/java/org/apache/zeppelin/rest/LoginRestApi.java
View file

@ -16,25 +16,39 @@
*/
package org.apache.zeppelin.rest;
import org.apache.shiro.authc.*;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Cookie;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.IncorrectCredentialsException;
import org.apache.shiro.authc.LockedAccountException;
import org.apache.shiro.authc.UnknownAccountException;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.realm.Realm;
import org.apache.shiro.subject.Subject;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.realm.jwt.JWTAuthenticationToken;
import org.apache.zeppelin.realm.jwt.KnoxJwtRealm;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.ticket.TicketContainer;
import org.apache.zeppelin.utils.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.FormParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
/**
* Created for org.apache.zeppelin.rest.message on 17/03/16.
*/
@ -42,6 +56,7 @@ import java.util.Map;
@Path("/login")
@Produces("application/json")
public class LoginRestApi {
private static final Logger LOG = LoggerFactory.getLogger(LoginRestApi.class);
/**
@ -52,6 +67,104 @@ public class LoginRestApi {
}
@GET
@ZeppelinApi
public Response getLogin(@Context HttpHeaders headers) {
JsonResponse response = null;
if (isKnoxSSOEnabled()) {
KnoxJwtRealm knoxJwtRealm = getJTWRealm();
Cookie cookie = headers.getCookies().get(knoxJwtRealm.getCookieName());
if (cookie != null && cookie.getValue() != null) {
Subject currentUser = org.apache.shiro.SecurityUtils.getSubject();
if (!currentUser.isAuthenticated()) {
JWTAuthenticationToken token = new JWTAuthenticationToken(null, cookie.getValue());
response = procedeToLogin(currentUser, token);
}
}
if (response == null) {
Map<String, String> data = new HashMap<>();
data.put("redirectURL", constructKnoxUrl(knoxJwtRealm, knoxJwtRealm.getLogin()));
response = new JsonResponse(Status.OK, "", data);
}
return response.build();
}
return new JsonResponse(Status.METHOD_NOT_ALLOWED).build();
}
private KnoxJwtRealm getJTWRealm() {
Collection realmsList = SecurityUtils.getRealmsList();
if (realmsList != null) {
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getClass().getName();
LOG.debug("RealmClass.getName: " + name);
if (name.equals("org.apache.zeppelin.realm.jwt.KnoxJwtRealm")) {
return (KnoxJwtRealm) realm;
}
}
}
return null;
}
private boolean isKnoxSSOEnabled() {
Collection realmsList = SecurityUtils.getRealmsList();
if (realmsList != null) {
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getClass().getName();
LOG.debug("RealmClass.getName: " + name);
if (name.equals("org.apache.zeppelin.realm.jwt.KnoxJwtRealm")) {
return true;
}
}
}
return false;
}
private JsonResponse procedeToLogin(Subject currentUser, AuthenticationToken token) {
JsonResponse response = null;
try {
currentUser.getSession().stop();
currentUser.getSession(true);
currentUser.login(token);
HashSet<String> roles = SecurityUtils.getRoles();
String principal = SecurityUtils.getPrincipal();
String ticket;
if ("anonymous".equals(principal)) {
ticket = "anonymous";
} else {
ticket = TicketContainer.instance.getTicket(principal);
}
Map<String, String> data = new HashMap<>();
data.put("principal", principal);
data.put("roles", roles.toString());
data.put("ticket", ticket);
response = new JsonResponse(Response.Status.OK, "", data);
//if no exception, that's it, we're done!
//set roles for user in NotebookAuthorization module
NotebookAuthorization.getInstance().setRoles(principal, roles);
} catch (UnknownAccountException uae) {
//username wasn't in the system, show them an error message?
LOG.error("Exception in login: ", uae);
} catch (IncorrectCredentialsException ice) {
//password didn't match, try again?
LOG.error("Exception in login: ", ice);
} catch (LockedAccountException lae) {
//account for that username is locked - can't login. Show them a message?
LOG.error("Exception in login: ", lae);
} catch (AuthenticationException ae) {
//unexpected condition - error?
LOG.error("Exception in login: ", ae);
}
return response;
}
/**
* Post Login
* Returns userName & password
@ -63,7 +176,7 @@ public class LoginRestApi {
@POST
@ZeppelinApi
public Response postLogin(@FormParam("userName") String userName,
@FormParam("password") String password) {
@FormParam("password") String password) {
JsonResponse response = null;
// ticket set to anonymous for anonymous user. Simplify testing.
Subject currentUser = org.apache.shiro.SecurityUtils.getSubject();
@ -71,45 +184,10 @@ public class LoginRestApi {
currentUser.logout();
}
if (!currentUser.isAuthenticated()) {
try {
UsernamePasswordToken token = new UsernamePasswordToken(userName, password);
// token.setRememberMe(true);
currentUser.getSession().stop();
currentUser.getSession(true);
currentUser.login(token);
UsernamePasswordToken token = new UsernamePasswordToken(userName, password);
HashSet<String> roles = SecurityUtils.getRoles();
String principal = SecurityUtils.getPrincipal();
String ticket;
if ("anonymous".equals(principal))
ticket = "anonymous";
else
ticket = TicketContainer.instance.getTicket(principal);
Map<String, String> data = new HashMap<>();
data.put("principal", principal);
data.put("roles", roles.toString());
data.put("ticket", ticket);
response = new JsonResponse(Response.Status.OK, "", data);
//if no exception, that's it, we're done!
//set roles for user in NotebookAuthorization module
NotebookAuthorization.getInstance().setRoles(principal, roles);
} catch (UnknownAccountException uae) {
//username wasn't in the system, show them an error message?
LOG.error("Exception in login: ", uae);
} catch (IncorrectCredentialsException ice) {
//password didn't match, try again?
LOG.error("Exception in login: ", ice);
} catch (LockedAccountException lae) {
//account for that username is locked - can't login. Show them a message?
LOG.error("Exception in login: ", lae);
} catch (AuthenticationException ae) {
//unexpected condition - error?
LOG.error("Exception in login: ", ae);
}
response = procedeToLogin(currentUser, token);
}
if (response == null) {
@ -129,9 +207,26 @@ public class LoginRestApi {
TicketContainer.instance.removeTicket(SecurityUtils.getPrincipal());
currentUser.getSession().stop();
currentUser.logout();
response = new JsonResponse(Response.Status.UNAUTHORIZED, "", "");
if (isKnoxSSOEnabled()) {
KnoxJwtRealm knoxJwtRealm = getJTWRealm();
Map<String, String> data = new HashMap<>();
data.put("redirectURL", constructKnoxUrl(knoxJwtRealm, knoxJwtRealm.getLogout()));
response = new JsonResponse(Status.UNAUTHORIZED, "", data);
} else {
response = new JsonResponse(Status.UNAUTHORIZED, "", "");
}
LOG.warn(response.toString());
return response.build();
}
private String constructKnoxUrl(KnoxJwtRealm knoxJwtRealm, String path) {
StringBuilder redirectURL = new StringBuilder(knoxJwtRealm.getProviderUrl());
redirectURL.append(path);
if (knoxJwtRealm.getRedirectParam() != null) {
redirectURL.append("?").append(knoxJwtRealm.getRedirectParam()).append("=");
}
return redirectURL.toString();
}
}

6
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -53,6 +53,8 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.socket.NotebookServer;
import org.apache.zeppelin.storage.ConfigStorage;
import org.apache.zeppelin.storage.FileSystemConfigStorage;
import org.apache.zeppelin.user.Credentials;
import org.apache.zeppelin.utils.SecurityUtils;
import org.eclipse.jetty.http.HttpVersion;
@ -82,6 +84,7 @@ public class ZeppelinServer extends Application {
private final InterpreterSettingManager interpreterSettingManager;
private SchedulerFactory schedulerFactory;
private InterpreterFactory replFactory;
private ConfigStorage configStorage;
private SearchService noteSearchService;
private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
@ -126,7 +129,7 @@ public class ZeppelinServer extends Application {
this.replFactory = new InterpreterFactory(interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
this.noteSearchService = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.init(conf);
this.notebookAuthorization = NotebookAuthorization.getInstance();
this.credentials = new Credentials(
conf.credentialsPersist(),
conf.getCredentialsPath(),
@ -134,6 +137,7 @@ public class ZeppelinServer extends Application {
notebook = new Notebook(conf,
notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
noteSearchService, notebookAuthorization, credentials);
this.configStorage = ConfigStorage.getInstance(conf);
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),

31
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -1108,6 +1108,13 @@ public class NotebookServer extends WebSocketServlet
}
Note note = notebook.getNote(noteId);
// drop cron
Map<String, Object> config = note.getConfig();
if (config.get("cron") != null) {
notebook.removeCron(note.getId());
}
if (note != null && !note.isTrash()){
fromMessage.put("name", Folder.TRASH_FOLDER_ID + "/" + note.getName());
renameNote(conn, userAndRoles, notebook, fromMessage, "move");
@ -1132,6 +1139,14 @@ public class NotebookServer extends WebSocketServlet
trashFolderId += Folder.TRASH_FOLDER_CONFLICT_INFIX + formatter.print(currentDate);
}
List<Note> noteList = folder.getNotesRecursively();
for (Note note: noteList) {
Map<String, Object> config = note.getConfig();
if (config.get("cron") != null) {
notebook.removeCron(note.getId());
}
}
fromMessage.put("name", trashFolderId);
renameFolder(conn, userAndRoles, notebook, fromMessage, "move");
}
@ -1147,6 +1162,13 @@ public class NotebookServer extends WebSocketServlet
}
Note note = notebook.getNote(noteId);
//restore cron
Map<String, Object> config = note.getConfig();
if (config.get("cron") != null) {
notebook.refreshCron(note.getId());
}
if (note != null && note.isTrash()) {
fromMessage.put("name", note.getName().replaceFirst(Folder.TRASH_FOLDER_ID + "/", ""));
renameNote(conn, userAndRoles, notebook, fromMessage, "restore");
@ -1166,6 +1188,15 @@ public class NotebookServer extends WebSocketServlet
if (folder != null && folder.isTrash()) {
String restoreName = folder.getId().replaceFirst(Folder.TRASH_FOLDER_ID + "/", "").trim();
//restore cron for each paragraph
List<Note> noteList = folder.getNotesRecursively();
for (Note note : noteList) {
Map<String, Object> config = note.getConfig();
if (config.get("cron") != null) {
notebook.refreshCron(note.getId());
}
}
// if the folder had conflict when it had moved to trash before
Pattern p = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
Matcher m = p.matcher(restoreName);

1
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -41,7 +41,6 @@ import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.realm.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.LdapRealm;
import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

224
zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java
View file

@ -1,224 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.webautomation;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.internal.Base64Encoder;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* from https://code.google.com/p/selenium/issues/detail?id=1361
*/
public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements TakesScreenshot {
private static Map<String, byte[]> imagesCache = Collections.synchronizedMap(new HashMap<String, byte[]>());
private static Map<String, String> cssjsCache = Collections.synchronizedMap(new HashMap<String, String>());
// http://stackoverflow.com/questions/4652777/java-regex-to-get-the-urls-from-css
private final static Pattern cssUrlPattern = Pattern.compile("background(-image)?[\\s]*:[^url]*url[\\s]*\\([\\s]*([^\\)]*)[\\s]*\\)[\\s]*");// ?<url>
static Logger LOGGER = LoggerFactory.getLogger(ScreenCaptureHtmlUnitDriver.class);
public ScreenCaptureHtmlUnitDriver() {
super();
}
public ScreenCaptureHtmlUnitDriver(boolean enableJavascript) {
super(enableJavascript);
}
public ScreenCaptureHtmlUnitDriver(Capabilities capabilities) {
super(capabilities);
}
public ScreenCaptureHtmlUnitDriver(BrowserVersion version) {
super(version);
DesiredCapabilities var = ((DesiredCapabilities) getCapabilities());
var.setCapability(CapabilityType.TAKES_SCREENSHOT, true);
}
@Override
@SuppressWarnings("unchecked")
public <X> X getScreenshotAs(OutputType<X> target) throws WebDriverException {
byte[] archive = new byte[0];
try {
archive = downloadCssAndImages(getWebClient(), (HtmlPage) getCurrentWindow().getEnclosedPage());
} catch (Exception e) {
LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while getScreenshotAs ", e);
}
if(target.equals(OutputType.BASE64)){
return target.convertFromBase64Png(new Base64Encoder().encode(archive));
}
if(target.equals(OutputType.FILE)){
File f = new File("screen.tmp");
try {
FileOutputStream scr = new FileOutputStream(f);
scr.write(archive);
scr.close();
} catch (IOException e) {
throw new WebDriverException(e);
}
return (X) f;
}
return (X) archive;
}
// http://stackoverflow.com/questions/2244272/how-can-i-tell-htmlunits-webclient-to-download-images-and-css
protected byte[] downloadCssAndImages(WebClient webClient, HtmlPage page) throws Exception {
WebWindow currentWindow = webClient.getCurrentWindow();
Map<String, String> urlMapping = new HashMap<>();
Map<String, byte[]> files = new HashMap<>();
WebWindow window = null;
try {
window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
webClient.getPage(window, new WebRequest(page.getUrl()));
} catch (Exception e) {
LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while downloadCssAndImages ", e);
window = webClient.openWindow(page.getUrl(), page.getUrl().toString()+"_screenshot");
}
String xPathExpression = "//*[name() = 'img' or name() = 'link' and (@type = 'text/css' or @type = 'image/x-icon') or @type = 'text/javascript']";
List<?> resultList = page.getByXPath(xPathExpression);
Iterator<?> i = resultList.iterator();
while (i.hasNext()) {
try {
HtmlElement el = (HtmlElement) i.next();
String resourceSourcePath = el.getAttribute("src").equals("") ? el.getAttribute("href") : el
.getAttribute("src");
if (resourceSourcePath == null || resourceSourcePath.equals(""))
continue;
URL resourceRemoteLink = page.getFullyQualifiedUrl(resourceSourcePath);
String resourceLocalPath = mapLocalUrl(page, resourceRemoteLink, resourceSourcePath, urlMapping);
urlMapping.put(resourceSourcePath, resourceLocalPath);
if (!resourceRemoteLink.toString().endsWith(".css")) {
byte[] image = downloadImage(webClient, window, resourceRemoteLink);
files.put(resourceLocalPath, image);
} else {
String css = downloadCss(webClient, window, resourceRemoteLink);
for (String cssImagePath : getLinksFromCss(css)) {
URL cssImagelink = page.getFullyQualifiedUrl(cssImagePath.replace("\"", "").replace("\'", "")
.replace(" ", ""));
String cssImageLocalPath = mapLocalUrl(page, cssImagelink, cssImagePath, urlMapping);
files.put(cssImageLocalPath, downloadImage(webClient, window, cssImagelink));
}
files.put(resourceLocalPath, replaceRemoteUrlsWithLocal(css, urlMapping)
.replace("resources/", "./").getBytes());
}
} catch (Exception e) {
LOGGER.error("Exception in ScreenCaptureHtmlUnitDriver while resultList.iterator ", e);
}
}
String pagesrc = replaceRemoteUrlsWithLocal(page.getWebResponse().getContentAsString(), urlMapping);
files.put("page.html", pagesrc.getBytes());
webClient.setCurrentWindow(currentWindow);
return createZip(files);
}
String downloadCss(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (cssjsCache.get(resourceUrl.toString()) == null) {
cssjsCache.put(resourceUrl.toString(), webClient.getPage(window, new WebRequest(resourceUrl))
.getWebResponse().getContentAsString());
}
return cssjsCache.get(resourceUrl.toString());
}
byte[] downloadImage(WebClient webClient, WebWindow window, URL resourceUrl) throws Exception {
if (imagesCache.get(resourceUrl.toString()) == null) {
imagesCache.put(
resourceUrl.toString(),
IOUtils.toByteArray(webClient.getPage(window, new WebRequest(resourceUrl)).getWebResponse()
.getContentAsStream()));
}
return imagesCache.get(resourceUrl.toString());
}
public static byte[] createZip(Map<String, byte[]> files) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ZipOutputStream zipfile = new ZipOutputStream(bos);
Iterator<String> i = files.keySet().iterator();
String fileName = null;
ZipEntry zipentry = null;
while (i.hasNext()) {
fileName = i.next();
zipentry = new ZipEntry(fileName);
zipfile.putNextEntry(zipentry);
zipfile.write(files.get(fileName));
}
zipfile.close();
return bos.toByteArray();
}
List<String> getLinksFromCss(String css) {
List<String> result = new LinkedList<>();
Matcher m = cssUrlPattern.matcher(css);
while (m.find()) { // find next match
result.add( m.group(2));
}
return result;
}
String replaceRemoteUrlsWithLocal(String source, Map<String, String> replacement) {
for (String object : replacement.keySet()) {
// background:url(http://org.com/images/image.gif)
source = source.replace(object, replacement.get(object));
}
return source;
}
String mapLocalUrl(HtmlPage page, URL link, String path, Map<String, String> replacementToAdd) throws Exception {
String resultingFileName = "resources/" + FilenameUtils.getName(link.getFile());
replacementToAdd.put(path, resultingFileName);
return resultingFileName;
}
}

16
zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinITUtils.java
View file

@ -41,20 +41,4 @@ public class ZeppelinITUtils {
LOG.info("Finished.");
}
}
public static void restartZeppelin() {
CommandExecutor.executeCommandLocalHost("../bin/zeppelin-daemon.sh restart",
false, ProcessData.Types_Of_Data.OUTPUT);
//wait for server to start.
sleep(5000, false);
}
public static void turnOffImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(0, TimeUnit.SECONDS);
}
public static void turnOnImplicitWaits(WebDriver driver) {
driver.manage().timeouts().implicitlyWait(AbstractZeppelinIT.MAX_IMPLICIT_WAIT,
TimeUnit.SECONDS);
}
}

86
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.rest;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import java.io.File;
import java.io.IOException;
import java.lang.ref.WeakReference;
@ -27,7 +30,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Pattern;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
@ -54,10 +56,6 @@ import org.hamcrest.TypeSafeMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
public abstract class AbstractTestRestApi {
protected static final Logger LOG = LoggerFactory.getLogger(AbstractTestRestApi.class);
@ -90,6 +88,48 @@ public abstract class AbstractTestRestApi {
"/api/version = anon\n" +
"/** = authc";
private static String zeppelinShiroKnox =
"[users]\n" +
"admin = password1, admin\n" +
"user1 = password2, role1, role2\n" +
"[main]\n" +
"knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm\n" +
"knoxJwtRealm.providerUrl = https://domain.example.com/\n" +
"knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html\n" +
"knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout\n" +
"knoxJwtRealm.redirectParam = originalUrl\n" +
"knoxJwtRealm.cookieName = hadoop-jwt\n" +
"knoxJwtRealm.publicKeyPath = knox-sso.pem\n" +
"authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter\n" +
"sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" +
"securityManager.sessionManager = $sessionManager\n" +
"securityManager.sessionManager.globalSessionTimeout = 86400000\n" +
"shiro.loginUrl = /api/login\n" +
"[roles]\n" +
"admin = *\n" +
"[urls]\n" +
"/api/version = anon\n" +
"/** = authc";
private static File knoxSsoPem = null;
private static String KNOX_SSO_PEM =
"-----BEGIN CERTIFICATE-----\n"
+ "MIIChjCCAe+gAwIBAgIJALYrdDEXKwcqMA0GCSqGSIb3DQEBBQUAMIGEMQswCQYD\n"
+ "VQQGEwJVUzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEPMA0GA1UEChMG\n"
+ "SGFkb29wMQ0wCwYDVQQLEwRUZXN0MTcwNQYDVQQDEy5jdHItZTEzNS0xNTEyMDY5\n"
+ "MDMyOTc1LTU0NDctMDEtMDAwMDAyLmh3eC5zaXRlMB4XDTE3MTIwNDA5NTIwMFoX\n"
+ "DTE4MTIwNDA5NTIwMFowgYQxCzAJBgNVBAYTAlVTMQ0wCwYDVQQIEwRUZXN0MQ0w\n"
+ "CwYDVQQHEwRUZXN0MQ8wDQYDVQQKEwZIYWRvb3AxDTALBgNVBAsTBFRlc3QxNzA1\n"
+ "BgNVBAMTLmN0ci1lMTM1LTE1MTIwNjkwMzI5NzUtNTQ0Ny0wMS0wMDAwMDIuaHd4\n"
+ "LnNpdGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAILFoXdz3yCy2INncYM2\n"
+ "y72fYrONoQIxeeIzeJIibXLTuowSju90Q6aThSyUsQ6NEia2flnlKiCgINTNAodh\n"
+ "UPUVGyGT+NMrqJzzpXAll2UUa6gIUPnXYEzYNkMIpbQOAo5BAg7YamaidbPPiT3W\n"
+ "wAD1rWo3AMUY+nZJrAi4dEH5AgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAB0R07/lo\n"
+ "4hD+WeDEeyLTnsbFnPNXxBT1APMUmmuCjcky/19ZB8OphqTKIITONdOK/XHdjZHG\n"
+ "JDOfhBkVknL42lSi45ahUAPS2PZOlQL08MbS8xajP1faterm+aHcdwJVK9dK76RB\n"
+ "/bA8TFNPblPxavIOcd+R+RfFmT1YKfYIhco=\n"
+ "-----END CERTIFICATE-----";
protected static File zeppelinHome;
protected static File confDir;
@ -127,7 +167,7 @@ public abstract class AbstractTestRestApi {
}
};
private static void start(boolean withAuth, String testClassName) throws Exception {
private static void start(boolean withAuth, String testClassName, boolean withKnox) throws Exception {
if (!wasRunning) {
// copy the resources files to a temp folder
zeppelinHome = new File("..");
@ -156,7 +196,18 @@ public abstract class AbstractTestRestApi {
if (!shiroIni.exists()) {
shiroIni.createNewFile();
}
FileUtils.writeStringToFile(shiroIni, zeppelinShiro);
if (withKnox) {
FileUtils.writeStringToFile(shiroIni,
zeppelinShiroKnox.replaceAll("knox-sso.pem", confDir + "/knox-sso.pem"));
knoxSsoPem = new File(confDir, "knox-sso.pem");
if (!knoxSsoPem.exists()) {
knoxSsoPem.createNewFile();
}
FileUtils.writeStringToFile(knoxSsoPem, KNOX_SSO_PEM);
} else {
FileUtils.writeStringToFile(shiroIni, zeppelinShiro);
}
}
// exclude org.apache.zeppelin.rinterpreter.* for scala 2.11 test
@ -254,13 +305,17 @@ public abstract class AbstractTestRestApi {
}
}
}
protected static void startUpWithKnoxEnable(String testClassName) throws Exception {
start(true, testClassName, true);
}
protected static void startUpWithAuthenticationEnable(String testClassName) throws Exception {
start(true, testClassName);
start(true, testClassName, false);
}
protected static void startUp(String testClassName) throws Exception {
start(false, testClassName);
start(false, testClassName, false);
}
private static String getHostname() {
@ -315,6 +370,10 @@ public abstract class AbstractTestRestApi {
}
protected static void shutDown() throws Exception {
shutDown(true);
}
protected static void shutDown(final boolean deleteConfDir) throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
@ -352,7 +411,7 @@ public abstract class AbstractTestRestApi {
.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName());
}
if (!ZeppelinServer.notebook.getConf().isRecoveryEnabled()) {
if (deleteConfDir && !ZeppelinServer.notebook.getConf().isRecoveryEnabled()) {
// don't delete interpreter.json when recovery is enabled. otherwise the interpreter setting
// id will change after zeppelin restart, then we can not recover interpreter process
// properly
@ -383,6 +442,10 @@ public abstract class AbstractTestRestApi {
}
protected static GetMethod httpGet(String path, String user, String pwd) throws IOException {
return httpGet(path, user, pwd, StringUtils.EMPTY);
}
protected static GetMethod httpGet(String path, String user, String pwd, String cookies) throws IOException {
LOG.info("Connecting to {}", url + path);
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod(url + path);
@ -390,6 +453,9 @@ public abstract class AbstractTestRestApi {
if (userAndPasswordAreNotBlank(user, pwd)) {
getMethod.setRequestHeader("Cookie", "JSESSIONID="+ getCookie(user, pwd));
}
if (!StringUtils.isBlank(cookies)) {
getMethod.setRequestHeader("Cookie", getMethod.getResponseHeader("Cookie") + ";" + cookies);
}
httpClient.executeMethod(getMethod);
LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText());
return getMethod;

82
zeppelin-server/src/test/java/org/apache/zeppelin/rest/KnoxRestApiTest.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest;
import com.google.gson.Gson;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.httpclient.methods.GetMethod;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class KnoxRestApiTest extends AbstractTestRestApi {
private String KNOX_COOKIE = "hadoop-jwt=eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzcyI6IktOT1hTU08iLCJleHAiOjE1MTM3NDU1MDd9.E2cWQo2sq75h0G_9fc9nWkL0SFMI5x_-Z0Zzr0NzQ86X4jfxliWYjr0M17Bm9GfPHRRR66s7YuYXa6DLbB4fHE0cyOoQnkfJFpU_vr1xhy0_0URc5v-Gb829b9rxuQfjKe-37hqbUdkwww2q6QQETVMvzp0rQKprUClZujyDvh0;";
@Rule
public ErrorCollector collector = new ErrorCollector();
private static final Logger LOG = LoggerFactory.getLogger(KnoxRestApiTest.class);
Gson gson = new Gson();
@BeforeClass
public static void init() throws Exception {
AbstractTestRestApi.startUpWithKnoxEnable(KnoxRestApiTest.class.getSimpleName());
}
@AfterClass
public static void destroy() throws Exception {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {
}
@Test
public void testThatOtherUserCanAccessNoteIfPermissionNotSet() throws IOException {
GetMethod loginWithoutCookie = httpGet("/api/security/ticket");
Map result = gson.fromJson(loginWithoutCookie.getResponseBodyAsString(), Map.class);
collector.checkThat("Path is redirected to /login", loginWithoutCookie.getPath(),
CoreMatchers.containsString("login"));
collector.checkThat("Path is redirected to /login", loginWithoutCookie.getPath(),
CoreMatchers.containsString("login"));
collector.checkThat("response contains redirect URL",
((Map) result.get("body")).get("redirectURL").toString(), CoreMatchers.equalTo(
"https://domain.example.com/gateway/knoxsso/knoxauth/login.html?originalUrl="));
GetMethod loginWithCookie = httpGet("/api/security/ticket", "", "", KNOX_COOKIE);
result = gson.fromJson(loginWithCookie.getResponseBodyAsString(), Map.class);
collector.checkThat("User logged in as admin",
((Map) result.get("body")).get("principal").toString(), CoreMatchers.equalTo("admin"));
System.out.println(result);
}
}

52
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -19,6 +19,7 @@ package org.apache.zeppelin.rest;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
@ -199,7 +200,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
}.getType());
Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
assertEquals((String)resp2Body.get("name"), "Note " + clonedNoteId);
assertEquals(resp2Body.get("name"), "Note " + clonedNoteId);
get.releaseConnection();
//cleanup
@ -269,4 +270,53 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRunWithServerRestart() throws Exception {
Note note1 = ZeppelinServer.notebook.createNote(anonymous);
// 2 paragraphs
// P1:
// %python
// import time
// time.sleep(1)
// from __future__ import print_function
// print(user)
// P2:
// %python
// user='abc'
//
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Paragraph p2 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("%python import time\ntime.sleep(1)\nuser='abc'");
p2.setText("%python from __future__ import print_function\nprint(user)");
PostMethod post1 = httpPost("/notebook/job/" + note1.getId(), "");
assertThat(post1, isAllowed());
post1.releaseConnection();
PutMethod put = httpPut("/notebook/" + note1.getId() + "/clear", "");
LOG.info("test clear paragraph output response\n" + put.getResponseBodyAsString());
assertThat(put, isAllowed());
put.releaseConnection();
// restart server (while keeping interpreter configuration)
AbstractTestRestApi.shutDown(false);
startUp(NotebookRestApiTest.class.getSimpleName());
note1 = ZeppelinServer.notebook.getNote(note1.getId());
p1 = note1.getParagraph(p1.getId());
p2 = note1.getParagraph(p2.getId());
PostMethod post2 = httpPost("/notebook/job/" + note1.getId(), "");
assertThat(post2, isAllowed());
Map<String, Object> resp = gson.fromJson(post2.getResponseBodyAsString(),
new TypeToken<Map<String, Object>>() {}.getType());
assertEquals(resp.get("status"), "OK");
post2.releaseConnection();
assertEquals(Job.Status.FINISHED, p1.getStatus());
assertEquals(Job.Status.FINISHED, p2.getStatus());
assertNotNull(p2.getResult());
assertEquals("abc\n", p2.getResult().message().get(0).getData());
}
}

84
zeppelin-server/src/test/scala/org/apache/zeppelin/AbstractFunctionalSuite.scala
View file

@ -1,84 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin
import org.apache.zeppelin.AbstractFunctionalSuite.SERVER_ADDRESS
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.firefox.{FirefoxBinary, FirefoxDriver, FirefoxProfile}
import org.openqa.selenium.safari.SafariDriver
import org.scalatest.concurrent.Eventually._
import org.scalatest.time._
import org.scalatest.selenium.WebBrowser
import org.scalatest.{BeforeAndAfterAll, FunSuite, Suite}
import scala.sys.process._
import scala.util.Try
object AbstractFunctionalSuite {
val SERVER_ADDRESS = "http://localhost:8080"
}
class AbstractFunctionalSuite extends FunSuite with WebBrowser with BeforeAndAfterAll {
implicit val webDriver = getDriver()
override def beforeAll() = {
"../bin/zeppelin-daemon.sh start" !
eventually (timeout(Span(180, Seconds))) {
go to SERVER_ADDRESS
assert(find("welcome").isDefined)
}
}
override def nestedSuites =
List[Suite](new WelcomePageSuite).toIndexedSeq
override def afterAll() = {
"../bin/zeppelin-daemon.sh stop" !
webDriver.close()
}
def getDriver(): WebDriver = {
val possibleDrivers = List[() => WebDriver](safari, chrome, firefox)
val createdDriver = possibleDrivers.map(driverFactory => Try(driverFactory.apply())).find(_.isSuccess)
createdDriver match {
case Some(driver) => driver.get
case None => throw new RuntimeException("Could not initialize any driver")
}
}
def safari(): WebDriver = {
new SafariDriver()
}
def chrome(): WebDriver = {
new ChromeDriver()
}
def firefox(): WebDriver = {
val ffox: FirefoxBinary = new FirefoxBinary
if ("true" == System.getenv("TRAVIS")) {
ffox.setEnvironmentProperty("DISPLAY", ":99")
}
val profile: FirefoxProfile = new FirefoxProfile
new FirefoxDriver(ffox, profile)
}
}

11751
zeppelin-web/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

22
zeppelin-web/pom.xml
View file

@ -46,7 +46,6 @@
<!--plugin versions-->
<plugin.frontend.nodeDownloadRoot>https://nodejs.org/dist/</plugin.frontend.nodeDownloadRoot>
<plugin.frontend.npmDownloadRoot>http://registry.npmjs.org/npm/-/</plugin.frontend.npmDownloadRoot>
<plugin.frontend.yarnDownloadRoot>https://github.com/yarnpkg/yarn/releases/download/</plugin.frontend.yarnDownloadRoot>
</properties>
<build>
@ -67,28 +66,25 @@
<configuration>
<nodeDownloadRoot>${plugin.frontend.nodeDownloadRoot}</nodeDownloadRoot>
<npmDownloadRoot>${plugin.frontend.npmDownloadRoot}</npmDownloadRoot>
<yarnDownloadRoot>${plugin.frontend.yarnDownloadRoot}</yarnDownloadRoot>
</configuration>
<executions>
<execution>
<id>install node and yarn</id>
<id>install node</id>
<goals>
<goal>install-node-and-yarn</goal>
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<yarnVersion>${yarn.version}</yarnVersion>
<npmVersion>${npm.version}</npmVersion>
</configuration>
</execution>
<execution>
<id>yarn install</id>
<id>npm install</id>
<goals>
<goal>yarn</goal>
<goal>npm</goal>
</goals>
<configuration>
<skip>${web.e2e.enabled}</skip>
@ -97,9 +93,9 @@
</execution>
<execution>
<id>yarn build</id>
<id>npm build</id>
<goals>
<goal>yarn</goal>
<goal>npm</goal>
</goals>
<configuration>
<skip>${web.e2e.enabled}</skip>
@ -108,9 +104,9 @@
</execution>
<execution>
<id>yarn test</id>
<id>npm test</id>
<goals>
<goal>yarn</goal>
<goal>npm</goal>
</goals>
<phase>test</phase>
<configuration>
@ -120,9 +116,9 @@
</execution>
<execution>
<id>yarn e2e</id>
<id>npm e2e</id>
<goals>
<goal>yarn</goal>
<goal>npm</goal>
</goals>
<phase>integration-test</phase>
<configuration>

16
zeppelin-web/src/app/app.js
View file

@ -183,12 +183,16 @@ function auth () {
let config = (process.env.PROD) ? {headers: { 'X-Requested-With': 'XMLHttpRequest' }} : {}
return $http.get(baseUrlSrv.getRestApiBase() + '/security/ticket', config).then(function (response) {
zeppelinWebApp.run(function ($rootScope) {
$rootScope.ticket = angular.fromJson(response.data).body
$rootScope.ticket.screenUsername = $rootScope.ticket.principal
if ($rootScope.ticket.principal.indexOf('#Pac4j') === 0) {
let re = ', name=(.*?),'
$rootScope.ticket.screenUsername = $rootScope.ticket.principal.match(re)[1]
let res = angular.fromJson(response.data).body
if (res['redirectURL']) {
window.location.href = res['redirectURL'] + window.location.href
} else {
$rootScope.ticket = res
$rootScope.ticket.screenUsername = $rootScope.ticket.principal
if ($rootScope.ticket.principal.indexOf('#Pac4j') === 0) {
let re = ', name=(.*?),'
$rootScope.ticket.screenUsername = $rootScope.ticket.principal.match(re)[1]
}
}
})
}, function (errorResponse) {

13
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -16,14 +16,15 @@ limitations under the License.
<h3>
<div style="float: left; width: auto; max-width: 40%"
ng-controller="ElasticInputCtrl as input">
<input type="text" pu-elastic-input class="form-control2" placeholder="New name" style="min-width: 0px; max-width: 95%;"
ng-if="input.showEditor" ng-model="input.value" ng-escape="input.showEditor = false" focus-if="input.showEditor"
ng-blur="updateNoteName(input.value);input.showEditor = false;" ng-enter="updateNoteName(input.value);input.showEditor = false;" />
<p class="form-control-static2"
<input type="text" pu-elastic-input class="form-control2" placeholder="New name"
style="min-width: 0px; max-width: 85%; margin-left: 2em;"
ng-if="input.showEditor" ng-model="input.value" ng-escape="input.showEditor = false" focus-if="input.showEditor"
ng-blur="updateNoteName(input.value);input.showEditor = false;" ng-enter="updateNoteName(input.value);input.showEditor = false;" />
<p class="form-control-static2 reverse-ellipsis ellipsis"
tooltip-placement="bottom"
uib-tooltip={{noteName(note)}}
ng-click="input.showEditor = !revisionView; input.value = note.name"
ng-show="!input.showEditor">{{noteName(note)}}</p>
ng-show="!input.showEditor"><span>{{noteName(note)}}</span>></p>
</div>
<div style="float: left; padding-bottom: 10px">
<span class="labelBtn btn-group">
@ -254,7 +255,7 @@ limitations under the License.
data-toggle="dropdown"
ng-class="{ 'btn-info' : note.config.cron, 'btn-danger' : note.info.cron, 'btn-default' : !note.config.cron}"
tooltip-placement="bottom" uib-tooltip="Run scheduler"
ng-disabled="revisionView">
ng-disabled="revisionView || isTrash(note)">
<span class="fa fa-clock-o"></span> {{getCronOptionNameFromValue(note.config.cron)}}
</div>
<ul class="dropdown-menu" role="menu" style="width:300px">

9
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -83,7 +83,14 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope,
let currentSearchParagraph = 0
$scope.$watch('note', function (value) {
$rootScope.pageTitle = value ? value.name : 'Zeppelin'
let title
if (value) {
title = value.name.substr(value.name.lastIndexOf('/') + 1, value.name.length)
title += ' - Zeppelin'
} else {
title = 'Zeppelin'
}
$rootScope.pageTitle = title
}, true)
$scope.$on('setConnectedStatus', function (event, param) {

47
zeppelin-web/src/app/notebook/notebook.css
View file

@ -183,6 +183,53 @@
overflow: hidden;
}
.ellipsis {
padding-left: 1em;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.reverse-ellipsis {
/* Your move. */
text-overflow: clip;
position: relative;
background-color: #FFF;
}
.reverse-ellipsis:before {
content: '\02026';
position: absolute;
z-index: 1;
left: -1em;
background-color: inherit;
padding-left: 1em;
margin-left: 0.5em;
}
.reverse-ellipsis span {
min-width: 100%;
position: relative;
display: inline-block;
float: right;
overflow: visible;
background-color: inherit;
text-indent: 0.5em;
}
.reverse-ellipsis span:before {
content: '';
position: absolute;
display: inline-block;
width: 1em;
height: 1em;
background-color: inherit;
z-index: 200;
left: -.5em;
}
.noOverflow {
overflow: hidden !important;
}

31
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -12,12 +12,11 @@
* limitations under the License.
*/
import { SpellResult, } from '../../spell'
import {
ParagraphStatus, isParagraphRunning,
} from './paragraph.status'
import {SpellResult} from '../../spell'
import {isParagraphRunning, ParagraphStatus} from './paragraph.status'
import moment from 'moment'
require('moment-duration-format')
const ParagraphExecutor = {
@ -1469,6 +1468,30 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca
}
})
$scope.$on('appendParagraphOutput', function (event, data) {
if (data.paragraphId === $scope.paragraph.id) {
if (!$scope.paragraph.results) {
$scope.paragraph.results = {}
if (!$scope.paragraph.results.msg) {
$scope.paragraph.results.msg = []
}
$scope.paragraph.results.msg[data.index] = {
data: data.data,
type: data.type
}
$rootScope.$broadcast(
'updateResult',
$scope.paragraph.results.msg[data.index],
$scope.paragraph.config.results[data.index],
$scope.paragraph,
data.index)
}
}
})
$scope.$on('keyEvent', function (event, keyEvent) {
if ($scope.paragraphFocused) {
let paragraphId = $scope.paragraph.id

44
zeppelin-web/src/app/notebook/paragraph/result/result.controller.js
View file

@ -22,11 +22,8 @@ import AreachartVisualization from '../../../visualization/builtins/visualizatio
import LinechartVisualization from '../../../visualization/builtins/visualization-linechart'
import ScatterchartVisualization from '../../../visualization/builtins/visualization-scatterchart'
import NetworkVisualization from '../../../visualization/builtins/visualization-d3network'
import {
DefaultDisplayType,
SpellResult,
} from '../../../spell'
import { ParagraphStatus, } from '../paragraph.status'
import {DefaultDisplayType, SpellResult} from '../../../spell'
import {ParagraphStatus} from '../paragraph.status'
const AnsiUp = require('ansi_up')
const AnsiUpConverter = new AnsiUp.default // eslint-disable-line new-parens,new-cap
@ -319,7 +316,7 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio
} else if (type === DefaultDisplayType.ANGULAR) {
renderAngular(targetElemId, data)
} else if (type === DefaultDisplayType.TEXT) {
renderText(targetElemId, data)
renderText(targetElemId, data, refresh)
} else if (type === DefaultDisplayType.ELEMENT) {
renderElem(targetElemId, data)
} else {
@ -464,7 +461,26 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio
return `p${resultId}_text`
}
const renderText = function (targetElemId, data) {
const checkAndReplaceCarriageReturn = function (str) {
if (/\r/.test(str)) {
let newGenerated = ''
let strArr = str.split('\n')
for (let str of strArr) {
if (/\r/.test(str)) {
let splitCR = str.split('\r')
newGenerated += splitCR[splitCR.length - 1] + '\n'
} else {
newGenerated += str + '\n'
}
}
// remove last "\n" character
return newGenerated.slice(0, -1)
} else {
return str
}
}
const renderText = function (targetElemId, data, refresh) {
const elem = angular.element(`#${targetElemId}`)
handleData(data, DefaultDisplayType.TEXT,
(generated) => {
@ -472,9 +488,16 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio
removeChildrenDOM(targetElemId)
if (generated) {
generated = checkAndReplaceCarriageReturn(generated)
const escaped = AnsiUpConverter.ansi_to_html(generated)
const divDOM = angular.element('<div></div>').innerHTML = escaped
elem.append(divDOM)
if (refresh) {
elem.html(divDOM)
} else {
elem.append(divDOM)
}
} else if (refresh) {
elem.html('')
}
elem.bind('mousewheel', (e) => { $scope.keepScrollDown = false })
@ -503,9 +526,8 @@ function ResultCtrl ($scope, $rootScope, $route, $window, $routeParams, $locatio
// pop all stacked data and append to the DOM
while (textResultQueueForAppend.length > 0) {
const line = textResultQueueForAppend.pop()
elem.append(angular.element('<div></div>').text(line))
const line = elem.html() + AnsiUpConverter.ansi_to_html(textResultQueueForAppend.pop())
elem.html(checkAndReplaceCarriageReturn(line))
if ($scope.keepScrollDown) {
const doc = angular.element(`#${elemId}`)
doc[0].scrollTop = doc[0].scrollHeight

104
zeppelin-web/src/app/tabledata/networkdata.js
View file

@ -40,42 +40,40 @@ export default class NetworkData extends TableData {
return
}
this.setNodesDefaults()
this.setEdgesDefaults()
this.graph.edges = this.graph.edges || []
this.networkNodes = angular.equals({}, this.graph.labels || {})
? null : {count: this.graph.nodes.length, labels: this.graph.labels}
this.networkRelationships = angular.equals([], this.graph.types || [])
? null : {count: this.graph.edges.length, types: this.graph.types}
let rows = []
let comment = ''
let entities = this.graph.nodes.concat(this.graph.edges)
let baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'},
{name: 'label', index: 1, aggr: 'sum'}]
let internalFieldsToJump = ['count', 'size', 'totalCount',
'data', 'x', 'y', 'labels']
let baseCols = _.map(baseColumnNames, function(col) { return col.name })
let keys = _.map(entities, function(elem) { return Object.keys(elem.data || {}) })
const rows = []
const comment = ''
const entities = this.graph.nodes.concat(this.graph.edges)
const baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'}]
const containsLabelField = _.find(entities, (entity) => 'label' in entity) != null
if (this.graph.labels || this.graph.types || containsLabelField) {
baseColumnNames.push({name: 'label', index: 1, aggr: 'sum'})
}
const internalFieldsToJump = ['count', 'size', 'totalCount',
'data', 'x', 'y', 'labels', 'source', 'target']
const baseCols = _.map(baseColumnNames, (col) => col.name)
let keys = _.map(entities, (elem) => Object.keys(elem.data || {}))
keys = _.flatten(keys)
keys = _.uniq(keys).filter(function(key) {
return baseCols.indexOf(key) === -1
})
let columnNames = baseColumnNames.concat(_.map(keys, function(elem, i) {
keys = _.uniq(keys).filter((key) => baseCols.indexOf(key) === -1)
const entityColumnNames = _.map(keys, (elem, i) => {
return {name: elem, index: i + baseColumnNames.length, aggr: 'sum'}
}))
})
const columnNames = baseColumnNames.concat(entityColumnNames)
for (let i = 0; i < entities.length; i++) {
let entity = entities[i]
let col = []
let col2 = []
const entity = entities[i]
const col = []
entity.data = entity.data || {}
for (let j = 0; j < columnNames.length; j++) {
let name = columnNames[j].name
let value = name in entity && internalFieldsToJump.indexOf(name) === -1
const name = columnNames[j].name
const value = name in entity && internalFieldsToJump.indexOf(name) === -1
? entity[name] : entity.data[name]
let parsedValue = value === null || value === undefined ? '' : value
const parsedValue = value === null || value === undefined ? '' : value
col.push(parsedValue)
col2.push({key: name, value: parsedValue})
}
rows.push(col)
}
@ -84,62 +82,4 @@ export default class NetworkData extends TableData {
this.columns = columnNames
this.rows = rows
}
setNodesDefaults() {
}
setEdgesDefaults() {
this.graph.edges
.sort((a, b) => {
if (a.source > b.source) {
return 1
} else if (a.source < b.source) {
return -1
} else if (a.target > b.target) {
return 1
} else if (a.target < b.target) {
return -1
} else {
return 0
}
})
this.graph.edges
.forEach((edge, index) => {
let prevEdge = this.graph.edges[index - 1]
edge.count = (index > 0 && +edge.source === +prevEdge.source && +edge.target === +prevEdge.target
? prevEdge.count : 0) + 1
edge.totalCount = this.graph.edges
.filter((innerEdge) => +edge.source === +innerEdge.source && +edge.target === +innerEdge.target)
.length
})
this.graph.edges
.forEach((edge) => {
if (typeof +edge.source === 'number') {
edge.source = this.graph.nodes.filter((node) => +edge.source === +node.id)[0] || null
}
if (typeof +edge.target === 'number') {
edge.target = this.graph.nodes.filter((node) => +edge.target === +node.id)[0] || null
}
})
}
getNetworkProperties() {
let baseCols = ['id', 'label']
let properties = {}
this.graph.nodes.forEach(function(node) {
let hasLabel = 'label' in node && node.label !== ''
if (!hasLabel) {
return
}
let label = node.label
let hasKey = hasLabel && label in properties
let keys = _.uniq(Object.keys(node.data || {})
.concat(hasKey ? properties[label].keys : baseCols))
if (!hasKey) {
properties[label] = {selected: 'label'}
}
properties[label].keys = keys
})
return properties
}
}

27
zeppelin-web/src/app/tabledata/networkdata.test.js
View file

@ -35,12 +35,33 @@ describe('NetworkData build', function() {
msg: JSON.stringify(jsonExpected)
})
expect(nd.columns.length).toBe(2)
expect(nd.columns.length).toBe(1)
expect(nd.rows.length).toBe(3)
expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id)
expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id)
expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id)
expect(nd.graph.edges[0].source.id).toBe(jsonExpected.nodes[1].id)
expect(nd.graph.edges[0].target.id).toBe(jsonExpected.nodes[0].id)
expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source)
expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target)
})
it('should able to show data fields source and target', function() {
let jsonExpected = {nodes: [{id: 1, data: {source: 'Source'}}, {id: 2, data: {target: 'Target'}}],
edges: [{source: 2, target: 1, id: 1, data: {source: 'Source Edge Data', target: 'Target Edge Data'}}]}
nd.loadParagraphResult({
type: DatasetType.NETWORK,
msg: JSON.stringify(jsonExpected)
})
expect(nd.columns.length).toBe(3)
expect(nd.rows.length).toBe(3)
expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id)
expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id)
expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id)
expect(nd.graph.edges[0].source).toBe(jsonExpected.edges[0].source)
expect(nd.graph.edges[0].target).toBe(jsonExpected.edges[0].target)
expect(nd.graph.nodes[0].data.source).toBe(jsonExpected.nodes[0].data.source)
expect(nd.graph.nodes[1].data.target).toBe(jsonExpected.nodes[1].data.target)
expect(nd.graph.edges[0].data.source).toBe(jsonExpected.edges[0].data.source)
expect(nd.graph.edges[0].data.target).toBe(jsonExpected.edges[0].data.target)
})
})

135
zeppelin-web/src/app/visualization/builtins/visualization-d3network.js
View file

@ -55,25 +55,36 @@ export default class NetworkVisualization extends Visualization {
console.log('graph not found')
return
}
console.log('Render Graph Visualization')
if (!networkData.isRendered) {
networkData.isRendered = true
} else {
return
}
console.log('Rendering the graph')
let transformationConfig = this.transformation.getSetting().scope.config
if (networkData.graph.edges.length &&
!networkData.isDefaultSet) {
networkData.isDefaultSet = true
this._setEdgesDefaults(networkData.graph)
}
const transformationConfig = this.transformation.getSetting().scope.config
console.log('cfg', transformationConfig)
if (transformationConfig && angular.equals({}, transformationConfig.properties)) {
transformationConfig.properties = networkData.getNetworkProperties()
transformationConfig.properties = this.getNetworkProperties(networkData.graph)
}
this.targetEl.empty().append('<svg></svg>')
let width = this.targetEl.width()
let height = this.targetEl.height()
let self = this
let defaultOpacity = 0
let nodeSize = 10
let textOffset = 3
let linkSize = 10
const width = this.targetEl.width()
const height = this.targetEl.height()
const self = this
const defaultOpacity = 0
const nodeSize = 10
const textOffset = 3
const linkSize = 10
let arcPath = (leftHand, d) => {
const arcPath = (leftHand, d) => {
let start = leftHand ? d.source : d.target
let end = leftHand ? d.target : d.source
let dx = end.x - start.x
@ -84,7 +95,7 @@ export default class NetworkVisualization extends Visualization {
return `M${start.x},${start.y}A${dr},${dr} 0 0,${sweep} ${end.x},${end.y}`
}
// Use elliptical arc path segments to doubly-encode directionality.
let tick = () => {
const tick = () => {
// Links
linkPath.attr('d', function(d) {
return arcPath(true, d)
@ -97,7 +108,7 @@ export default class NetworkVisualization extends Visualization {
text.attr('transform', (d) => `translate(${d.x},${d.y})`)
}
let setOpacity = (scale) => {
const setOpacity = (scale) => {
let opacity = scale >= +transformationConfig.d3Graph.zoom.minScale ? 1 : 0
this.svg.selectAll('.nodeLabel')
.style('opacity', opacity)
@ -105,7 +116,7 @@ export default class NetworkVisualization extends Visualization {
.style('opacity', opacity)
}
let zoom = d3.behavior.zoom()
const zoom = d3.behavior.zoom()
.scaleExtent([1, 10])
.on('zoom', () => {
console.log('zoom')
@ -135,13 +146,15 @@ export default class NetworkVisualization extends Visualization {
})
.start()
let renderFooterOnClick = (entity, type) => {
let footerId = this.containerId + '_footer'
let obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type}
let html = [this.$interpolate(['<li><b>{{type}}_id:</b>&nbsp{{id}}</li>',
'<li><b>{{type}}_type:</b>&nbsp{{label}}</li>'].join(''))(obj)]
const renderFooterOnClick = (entity, type) => {
const footerId = this.containerId + '_footer'
const obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type}
let html = [`<li><b>${obj.type}_id:</b>&nbsp${obj.id}</li>`]
if (obj.label) {
html.push(`<li><b>${obj.type}_type:</b>&nbsp${obj.label}</li>`)
}
html = html.concat(_.map(entity.data, (v, k) => {
return this.$interpolate('<li><b>{{field}}:</b>&nbsp{{value}}</li>')({field: k, value: v})
return `<li><b>${k}:</b>&nbsp${v}</li>`
}))
angular.element('#' + footerId)
.find('.list-inline')
@ -149,7 +162,7 @@ export default class NetworkVisualization extends Visualization {
.append(html.join(''))
}
let drag = d3.behavior.drag()
const drag = d3.behavior.drag()
.origin((d) => d)
.on('dragstart', function(d) {
console.log('dragstart')
@ -171,7 +184,7 @@ export default class NetworkVisualization extends Visualization {
self.force.resume()
})
let container = this.svg.append('g')
const container = this.svg.append('g')
if (networkData.graph.directed) {
container.append('svg:defs').selectAll('marker')
.data(['arrowMarker-' + this.containerId])
@ -188,7 +201,7 @@ export default class NetworkVisualization extends Visualization {
.attr('d', 'M0,-5L10,0L0,5')
}
// Links
let link = container.append('svg:g')
const link = container.append('svg:g')
.on('click', () => {
renderFooterOnClick(d3.select(d3.event.target).datum(), 'edge')
})
@ -196,13 +209,13 @@ export default class NetworkVisualization extends Visualization {
.data(self.force.links())
.enter()
.append('g')
let getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count
let showLabel = (d) => this._showNodeLabel(d)
let linkPath = link.append('svg:path')
const getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count
const showLabel = (d) => this._showNodeLabel(d)
const linkPath = link.append('svg:path')
.attr('class', 'link')
.attr('size', linkSize)
.attr('marker-end', `url(#arrowMarker-${this.containerId})`)
let textPath = link.append('svg:path')
const textPath = link.append('svg:path')
.attr('id', getPathId)
.attr('class', 'textpath')
container.append('svg:g')
@ -218,7 +231,7 @@ export default class NetworkVisualization extends Visualization {
.text((d) => d.label)
.style('opacity', defaultOpacity)
// Nodes
let circle = container.append('svg:g')
const circle = container.append('svg:g')
.on('click', () => {
renderFooterOnClick(d3.select(d3.event.target).datum(), 'node')
})
@ -229,7 +242,7 @@ export default class NetworkVisualization extends Visualization {
.attr('fill', (d) => networkData.graph.labels && d.label in networkData.graph.labels
? networkData.graph.labels[d.label] : '#000000')
.call(drag)
let text = container.append('svg:g').selectAll('g')
const text = container.append('svg:g').selectAll('g')
.data(self.force.nodes())
.enter().append('svg:g')
text.append('svg:text')
@ -252,12 +265,72 @@ export default class NetworkVisualization extends Visualization {
}
_showNodeLabel(d) {
let transformationConfig = this.transformation.getSetting().scope.config
let selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected
const transformationConfig = this.transformation.getSetting().scope.config
const selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected
return d.data[selectedLabel] || d[selectedLabel]
}
getTransformation() {
return this.transformation
}
setNodesDefaults() {
}
_setEdgesDefaults(graph) {
graph.edges
.sort((a, b) => {
if (a.source > b.source) {
return 1
} else if (a.source < b.source) {
return -1
} else if (a.target > b.target) {
return 1
} else if (a.target < b.target) {
return -1
} else {
return 0
}
})
graph.edges
.forEach((edge, index) => {
let prevEdge = graph.edges[index - 1]
edge.count = (index > 0 && +edge.source === +prevEdge.source && +edge.target === +prevEdge.target
? prevEdge.count : 0) + 1
edge.totalCount = graph.edges
.filter((innerEdge) => +edge.source === +innerEdge.source && +edge.target === +innerEdge.target)
.length
})
graph.edges
.forEach((edge) => {
if (typeof +edge.source === 'number') {
// edge.source = graph.nodes.filter((node) => +edge.source === +node.id)[0] || null
edge.source = _.find(graph.nodes, (node) => +edge.source === +node.id)
}
if (typeof +edge.target === 'number') {
// edge.target = graph.nodes.filter((node) => +edge.target === +node.id)[0] || null
edge.target = _.find(graph.nodes, (node) => +edge.target === +node.id)
}
})
}
getNetworkProperties(graph) {
const baseCols = ['id', 'label']
const properties = {}
graph.nodes.forEach(function(node) {
const hasLabel = 'label' in node && node.label !== ''
if (!hasLabel) {
return
}
const label = node.label
const hasKey = hasLabel && label in properties
const keys = _.uniq(Object.keys(node.data || {})
.concat(hasKey ? properties[label].keys : baseCols))
if (!hasKey) {
properties[label] = {selected: 'label'}
}
properties[label].keys = keys
})
return properties
}
}

76
zeppelin-web/src/components/navbar/navbar.controller.js
View file

@ -86,13 +86,55 @@ function NavCtrl ($scope, $rootScope, $http, $routeParams, $location,
websocketMsgSrv.getHomeNote()
}
function logout () {
function logout() {
let logoutURL = baseUrlSrv.getRestApiBase() + '/login/logout'
// for firefox and safari
logoutURL = logoutURL.replace('//', '//false:false@')
$http.post(logoutURL).error(function () {
$http.post(logoutURL).then(function () {}, function (response) {
if (response.data) {
let res = angular.fromJson(response.data).body
if (res['redirectURL']) {
window.location.href = res['redirectURL'] + window.location.href
}
}
// force authcBasic (if configured) to logout
if (detectIE()) {
let outcome
try {
outcome = document.execCommand('ClearAuthenticationCache')
} catch (e) {
console.log(e)
}
if (!outcome) {
// Let's create an xmlhttp object
outcome = (function (x) {
if (x) {
// the reason we use "random" value for password is
// that browsers cache requests. changing
// password effectively behaves like cache-busing.
x.open('HEAD', location.href, true, 'logout',
(new Date()).getTime().toString())
x.send('')
// x.abort()
return 1 // this is **speculative** "We are done."
} else {
// eslint-disable-next-line no-useless-return
return
}
})(window.XMLHttpRequest ? new window.XMLHttpRequest()
// eslint-disable-next-line no-undef
: (window.ActiveXObject ? new ActiveXObject('Microsoft.XMLHTTP') : u))
}
if (!outcome) {
let m = 'Your browser is too old or too weird to support log out functionality. Close all windows and ' +
'restart the browser.'
alert(m)
}
} else {
// for firefox and safari
logoutURL = logoutURL.replace('//', '//false:false@')
}
$http.post(logoutURL).error(function () {
$rootScope.userName = ''
$rootScope.ticket.principal = ''
@ -109,6 +151,32 @@ function NavCtrl ($scope, $rootScope, $http, $routeParams, $location,
})
}
function detectIE() {
let ua = window.navigator.userAgent
let msie = ua.indexOf('MSIE ')
if (msie > 0) {
// IE 10 or older => return version number
return parseInt(ua.substring(msie + 5, ua.indexOf('.', msie)), 10)
}
let trident = ua.indexOf('Trident/')
if (trident > 0) {
// IE 11 => return version number
let rv = ua.indexOf('rv:')
return parseInt(ua.substring(rv + 3, ua.indexOf('.', rv)), 10)
}
let edge = ua.indexOf('Edge/')
if (edge > 0) {
// Edge (IE 12+) => return version number
return parseInt(ua.substring(edge + 5, ua.indexOf('.', edge)), 10)
}
// other browser
return false
}
function search (searchTerm) {
$location.path('/search/' + searchTerm)
}

694
zeppelin-zengine/pom.xml
View file

@ -299,332 +299,6 @@
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-core</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-server</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-json</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-spark_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
@ -642,11 +316,373 @@
<systemProperties>
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
</systemProperties>
<environmentVariables>
<!--<ZEPPELIN_HOME>..</ZEPPELIN_HOME>-->
</environmentVariables>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>hadoop2</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-core</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-server</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<!--<exclusion>-->
<!--<groupId>com.sun.jersey</groupId>-->
<!--<artifactId>jersey-json</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>jackrabbit-webdav</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-spark_2.10</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
<profile>
<id>hadoop3</id>
<properties>
<hadoop.version>3.0.0</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</profile>
</profiles>
</project>

3
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file

@ -699,13 +699,14 @@ public class InterpreterSetting {
}
synchronized RemoteInterpreterProcess createInterpreterProcess(String interpreterGroupId,
String userName,
Properties properties)
throws IOException {
if (launcher == null) {
createLauncher();
}
InterpreterLaunchContext launchContext = new
InterpreterLaunchContext(properties, option, interpreterRunner,
InterpreterLaunchContext(properties, option, interpreterRunner, userName,
interpreterGroupId, id, group, name);
RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext);
process.setRemoteInterpreterEventPoller(

143
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
View file

@ -44,18 +44,20 @@ import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourceSet;
import org.apache.zeppelin.util.ReflectionUtils;
import org.apache.zeppelin.storage.ConfigStorage;
import org.apache.zeppelin.storage.FileSystemConfigStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.Proxy;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.Authentication;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
@ -75,6 +77,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* InterpreterSettingManager is the component which manage all the interpreter settings.
* (load/create/update/remove/get)
@ -90,7 +93,6 @@ public class InterpreterSettingManager {
private final ZeppelinConfiguration conf;
private final Path interpreterDirPath;
private final Path interpreterSettingPath;
/**
* This is only InterpreterSetting templates with default name and properties
@ -123,6 +125,9 @@ public class InterpreterSettingManager {
private DependencyResolver dependencyResolver;
private LifecycleManager lifecycleManager;
private RecoveryStorage recoveryStorage;
private ConfigStorage configStorage;
public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
AngularObjectRegistryListener angularObjectRegistryListener,
@ -133,22 +138,21 @@ public class InterpreterSettingManager {
this(zeppelinConfiguration, new InterpreterOption(),
angularObjectRegistryListener,
remoteInterpreterProcessListener,
appEventListener);
appEventListener,
ConfigStorage.getInstance(zeppelinConfiguration));
}
@VisibleForTesting
public InterpreterSettingManager(ZeppelinConfiguration conf,
InterpreterOption defaultOption,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener
remoteInterpreterProcessListener,
ApplicationEventListener appEventListener) throws IOException {
ApplicationEventListener appEventListener,
ConfigStorage configStorage) throws IOException {
this.conf = conf;
this.defaultOption = defaultOption;
this.interpreterDirPath = Paths.get(conf.getInterpreterDir());
LOGGER.debug("InterpreterRootPath: {}", interpreterDirPath);
this.interpreterSettingPath = Paths.get(conf.getInterpreterSettingPath());
LOGGER.debug("InterpreterSettingPath: {}", interpreterSettingPath);
this.dependencyResolver = new DependencyResolver(
conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
this.interpreterRepositories = dependencyResolver.getRepos();
@ -159,18 +163,18 @@ public class InterpreterSettingManager {
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
this.recoveryStorage = ReflectionUtils.createClazzInstance(conf.getRecoveryStorageClass(),
new Class[] {ZeppelinConfiguration.class, InterpreterSettingManager.class},
new Object[] {conf, this});
this.recoveryStorage.init();
LOGGER.info("Using RecoveryStorage: " + this.recoveryStorage.getClass().getName());
this.lifecycleManager = ReflectionUtils.createClazzInstance(conf.getLifecycleManagerClass(),
new Class[] {ZeppelinConfiguration.class},
new Object[] {conf});
LOGGER.info("Using LifecycleManager: " + this.lifecycleManager.getClass().getName());
this.configStorage = configStorage;
init();
}
@ -190,10 +194,11 @@ public class InterpreterSettingManager {
/**
* Load interpreter setting from interpreter-setting.json
*/
private void loadFromFile() {
if (!Files.exists(interpreterSettingPath)) {
private void loadFromFile() throws IOException {
InterpreterInfoSaving infoSaving =
configStorage.loadInterpreterSettings();
if (infoSaving == null) {
// nothing to read
LOGGER.warn("Interpreter Setting file {} doesn't exist", interpreterSettingPath);
for (InterpreterSetting interpreterSettingTemplate : interpreterSettingTemplates.values()) {
InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate);
initInterpreterSetting(interpreterSetting);
@ -202,71 +207,65 @@ public class InterpreterSettingManager {
return;
}
try {
InterpreterInfoSaving infoSaving = InterpreterInfoSaving.loadFromFile(interpreterSettingPath);
//TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) {
savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
savedInterpreterSetting.getProperties()
));
initInterpreterSetting(savedInterpreterSetting);
//TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) {
savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
savedInterpreterSetting.getProperties()
));
initInterpreterSetting(savedInterpreterSetting);
InterpreterSetting interpreterSettingTemplate =
interpreterSettingTemplates.get(savedInterpreterSetting.getGroup());
// InterpreterSettingTemplate is from interpreter-setting.json which represent the latest
// InterpreterSetting, while InterpreterSetting is from interpreter.json which represent
// the user saved interpreter setting
if (interpreterSettingTemplate != null) {
savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir());
// merge properties from interpreter-setting.json and interpreter.json
Map<String, InterpreterProperty> mergedProperties =
new HashMap<>(InterpreterSetting.convertInterpreterProperties(
interpreterSettingTemplate.getProperties()));
Map<String, InterpreterProperty> savedProperties = InterpreterSetting
.convertInterpreterProperties(savedInterpreterSetting.getProperties());
for (Map.Entry<String, InterpreterProperty> entry : savedProperties.entrySet()) {
// only merge properties whose value is not empty
if (entry.getValue().getValue() != null && !
StringUtils.isBlank(entry.getValue().toString())) {
mergedProperties.put(entry.getKey(), entry.getValue());
}
}
savedInterpreterSetting.setProperties(mergedProperties);
// merge InterpreterInfo
savedInterpreterSetting.setInterpreterInfos(
interpreterSettingTemplate.getInterpreterInfos());
savedInterpreterSetting.setInterpreterRunner(
interpreterSettingTemplate.getInterpreterRunner());
} else {
LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: "
+ savedInterpreterSetting.getGroup());
}
// Overwrite the default InterpreterSetting we registered from InterpreterSetting Templates
// remove it first
for (InterpreterSetting setting : interpreterSettings.values()) {
if (setting.getName().equals(savedInterpreterSetting.getName())) {
interpreterSettings.remove(setting.getId());
InterpreterSetting interpreterSettingTemplate =
interpreterSettingTemplates.get(savedInterpreterSetting.getGroup());
// InterpreterSettingTemplate is from interpreter-setting.json which represent the latest
// InterpreterSetting, while InterpreterSetting is from interpreter.json which represent
// the user saved interpreter setting
if (interpreterSettingTemplate != null) {
savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir());
// merge properties from interpreter-setting.json and interpreter.json
Map<String, InterpreterProperty> mergedProperties =
new HashMap<>(InterpreterSetting.convertInterpreterProperties(
interpreterSettingTemplate.getProperties()));
Map<String, InterpreterProperty> savedProperties = InterpreterSetting
.convertInterpreterProperties(savedInterpreterSetting.getProperties());
for (Map.Entry<String, InterpreterProperty> entry : savedProperties.entrySet()) {
// only merge properties whose value is not empty
if (entry.getValue().getValue() != null && !
StringUtils.isBlank(entry.getValue().toString())) {
mergedProperties.put(entry.getKey(), entry.getValue());
}
}
savedInterpreterSetting.postProcessing();
LOGGER.info("Create Interpreter Setting {} from interpreter.json",
savedInterpreterSetting.getName());
interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting);
savedInterpreterSetting.setProperties(mergedProperties);
// merge InterpreterInfo
savedInterpreterSetting.setInterpreterInfos(
interpreterSettingTemplate.getInterpreterInfos());
savedInterpreterSetting.setInterpreterRunner(
interpreterSettingTemplate.getInterpreterRunner());
} else {
LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: "
+ savedInterpreterSetting.getGroup());
}
interpreterBindings.putAll(infoSaving.interpreterBindings);
if (infoSaving.interpreterRepositories != null) {
for (RemoteRepository repo : infoSaving.interpreterRepositories) {
if (!dependencyResolver.getRepos().contains(repo)) {
this.interpreterRepositories.add(repo);
}
// Overwrite the default InterpreterSetting we registered from InterpreterSetting Templates
// remove it first
for (InterpreterSetting setting : interpreterSettings.values()) {
if (setting.getName().equals(savedInterpreterSetting.getName())) {
interpreterSettings.remove(setting.getId());
}
}
savedInterpreterSetting.postProcessing();
LOGGER.info("Create Interpreter Setting {} from interpreter.json",
savedInterpreterSetting.getName());
interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting);
}
interpreterBindings.putAll(infoSaving.interpreterBindings);
if (infoSaving.interpreterRepositories != null) {
for (RemoteRepository repo : infoSaving.interpreterRepositories) {
if (!dependencyResolver.getRepos().contains(repo)) {
this.interpreterRepositories.add(repo);
}
}
} catch (IOException e) {
LOGGER.error("Fail to load interpreter setting configuration file: "
+ interpreterSettingPath, e);
}
}
@ -276,7 +275,7 @@ public class InterpreterSettingManager {
info.interpreterBindings = interpreterBindings;
info.interpreterSettings = interpreterSettings;
info.interpreterRepositories = interpreterRepositories;
info.saveToFile(interpreterSettingPath);
configStorage.save(info);
}
}

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java
View file

@ -60,10 +60,11 @@ public class ManagedInterpreterGroup extends InterpreterGroup {
throws IOException {
if (remoteInterpreterProcess == null) {
LOGGER.info("Create InterpreterProcess for InterpreterGroup: " + getId());
remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(id, properties);
remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(id, userName,
properties);
synchronized (remoteInterpreterProcess) {
if (!remoteInterpreterProcess.isRunning()) {
remoteInterpreterProcess.start(userName, false);
remoteInterpreterProcess.start(userName);
remoteInterpreterProcess.getRemoteInterpreterEventPoller()
.setInterpreterProcess(remoteInterpreterProcess);
remoteInterpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(this);

8
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
View file

@ -84,15 +84,15 @@ public class ShellScriptLauncher extends InterpreterLauncher {
runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(),
zConf.getCallbackPortRange(), zConf.getInterpreterPortRange(),
zConf.getInterpreterDir() + "/" + groupName, localRepoPath,
buildEnvFromProperties(), connectTimeout, name);
buildEnvFromProperties(context), connectTimeout, name, option.isUserImpersonate());
}
}
protected Map<String, String> buildEnvFromProperties() {
protected Map<String, String> buildEnvFromProperties(InterpreterLaunchContext context) {
Map<String, String> env = new HashMap<>();
for (Object key : properties.keySet()) {
for (Object key : context.getProperties().keySet()) {
if (RemoteInterpreterUtils.isEnvString((String) key)) {
env.put((String) key, properties.getProperty((String) key));
env.put((String) key, context.getProperties().getProperty((String) key));
}
}
return env;

13
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
View file

@ -41,7 +41,7 @@ public class SparkInterpreterLauncher extends ShellScriptLauncher {
}
@Override
protected Map<String, String> buildEnvFromProperties() {
protected Map<String, String> buildEnvFromProperties(InterpreterLaunchContext context) {
Map<String, String> env = new HashMap<String, String>();
Properties sparkProperties = new Properties();
String sparkMaster = getSparkMaster(properties);
@ -70,6 +70,11 @@ public class SparkInterpreterLauncher extends ShellScriptLauncher {
for (String name : sparkProperties.stringPropertyNames()) {
sparkConfBuilder.append(" --conf " + name + "=" + sparkProperties.getProperty(name));
}
String useProxyUserEnv = System.getenv("ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER");
if (context.getOption().isUserImpersonate() && (StringUtils.isBlank(useProxyUserEnv) ||
!useProxyUserEnv.equals("false"))) {
sparkConfBuilder.append(" --proxy-user " + context.getUserName());
}
env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
@ -194,12 +199,12 @@ public class SparkInterpreterLauncher extends ShellScriptLauncher {
}
private String toShellFormat(String value) {
if (value.contains("\'") && value.contains("\"")) {
if (value.contains("'") && value.contains("\"")) {
throw new RuntimeException("Spark property value could not contain both \" and '");
} else if (value.contains("\'")) {
} else if (value.contains("'")) {
return "\"" + value + "\"";
} else {
return "\'" + value + "\'";
return "'" + value + "'";
}
}

14
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
View file

@ -63,6 +63,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
private final String interpreterDir;
private final String localRepoDir;
private final String interpreterSettingName;
private final boolean isUserImpersonated;
private Map<String, String> env;
@ -74,7 +75,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
String localRepoDir,
Map<String, String> env,
int connectTimeout,
String interpreterSettingName) {
String interpreterSettingName,
boolean isUserImpersonated) {
super(connectTimeout);
this.interpreterRunner = intpRunner;
this.callbackPortRange = callbackPortRange;
@ -83,6 +85,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
this.interpreterSettingName = interpreterSettingName;
this.isUserImpersonated = isUserImpersonated;
}
@Override
@ -96,7 +99,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
}
@Override
public void start(String userName, Boolean isUserImpersonate) {
public void start(String userName) {
// start server process
final String callbackHost;
final int callbackPort;
@ -161,7 +164,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
cmdLine.addArgument(Integer.toString(callbackPort), false);
cmdLine.addArgument("-r", false);
cmdLine.addArgument(interpreterPortRange, false);
if (isUserImpersonate && !userName.equals("anonymous")) {
if (isUserImpersonated && !userName.equals("anonymous")) {
cmdLine.addArgument("-u", false);
cmdLine.addArgument(userName, false);
}
@ -272,6 +275,11 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
return interpreterRunner;
}
@VisibleForTesting
public boolean isUserImpersonated() {
return isUserImpersonated;
}
public boolean isRunning() {
return running.get();
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
View file

@ -58,7 +58,7 @@ public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess {
}
@Override
public void start(String userName, Boolean isUserImpersonate) {
public void start(String userName) {
// assume process is externally managed. nothing to do
}

13
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/FileSystemStorage.java
View file

@ -40,6 +40,8 @@ public class FileSystemStorage {
private FileSystemStorage(ZeppelinConfiguration zConf) throws IOException {
this.zConf = zConf;
this.hadoopConf = new Configuration();
// disable checksum for local file system. because interpreter.json may be updated by
// no hadoop filesystem api
this.hadoopConf.set("fs.file.impl", RawLocalFileSystem.class.getName());
this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
@ -74,6 +76,16 @@ public class FileSystemStorage {
return fs.makeQualified(path);
}
public boolean exists(final Path path) throws IOException {
return callHdfsOperation(new HdfsOperation<Boolean>() {
@Override
public Boolean call() throws IOException {
return fs.exists(path);
}
});
}
public void tryMkDir(final Path dir) throws IOException {
callHdfsOperation(new HdfsOperation<Void>() {
@Override
@ -149,7 +161,6 @@ public class FileSystemStorage {
public synchronized <T> T callHdfsOperation(final HdfsOperation<T> func) throws IOException {
if (isSecurityEnabled) {
UserGroupInformation.getLoginUser().reloginFromKeytab();
try {
return UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<T>() {
@Override

45
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -20,13 +20,17 @@ package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
import java.util.*;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.GsonBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
import org.apache.zeppelin.completer.CompletionType;
@ -34,7 +38,13 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
@ -47,8 +57,10 @@ import org.apache.zeppelin.user.Credentials;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* Binded interpreters for a note
@ -60,7 +72,8 @@ public class Note implements ParagraphJobListener, JsonSerializable {
.setPrettyPrinting()
.setDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
.registerTypeAdapter(Date.class, new NotebookImportDeserializer())
.registerTypeAdapterFactory(Input.TypeAdapterFactory).create();
.registerTypeAdapterFactory(Input.TypeAdapterFactory)
.create();
// threadpool for delayed persist of note
private static final ScheduledThreadPoolExecutor delayedPersistThreadPool =
@ -652,6 +665,22 @@ public class Note implements ParagraphJobListener, JsonSerializable {
return true;
}
/**
* Return true if there is a running or pending paragraph
*/
boolean isRunningOrPending() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
Status status = p.getStatus();
if (status.isRunning() || status.isPending()) {
return true;
}
}
}
return false;
}
public boolean isTrash() {
String path = getName();
if (path.charAt(0) == '/') {
@ -918,6 +947,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
return !interpreterSettingManager.getInterpreterSettings(getId()).isEmpty();
}
@Override
public String toJson() {
return gson.toJson(this);
}
@ -925,13 +955,14 @@ public class Note implements ParagraphJobListener, JsonSerializable {
public static Note fromJson(String json) {
Note note = gson.fromJson(json, Note.class);
convertOldInput(note);
note.resetRuntimeInfos();
note.postProcessParagraphs();
return note;
}
public void resetRuntimeInfos() {
public void postProcessParagraphs() {
for (Paragraph p : paragraphs) {
p.clearRuntimeInfos();
p.parseText();
}
}

15
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -891,16 +891,15 @@ public class Notebook implements NoteEventListener {
String noteId = context.getJobDetail().getJobDataMap().getString("noteId");
Note note = notebook.getNote(noteId);
note.runAll();
while (!note.isTerminated()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.error(e.toString(), e);
}
if (note.isRunningOrPending()) {
logger.warn("execution of the cron job is skipped because there is a running or pending " +
"paragraph (note id: {})", noteId);
return;
}
note.runAll();
boolean releaseResource = false;
String cronExecutingUser = null;
try {
@ -975,7 +974,7 @@ public class Notebook implements NoteEventListener {
}
}
private void removeCron(String id) {
public void removeCron(String id) {
try {
quartzSched.deleteJob(new JobKey(id, "note"));
} catch (SchedulerException e) {

53
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookAuthorization.java
View file

@ -35,6 +35,7 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.storage.ConfigStorage;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,8 +62,8 @@ public class NotebookAuthorization {
*/
private static Map<String, Set<String>> userRoles = new HashMap<>();
private static ZeppelinConfiguration conf;
private static Gson gson;
private static String filePath;
private static ConfigStorage configStorage;
private NotebookAuthorization() {}
@ -70,11 +71,8 @@ public class NotebookAuthorization {
if (instance == null) {
instance = new NotebookAuthorization();
conf = config;
filePath = conf.getNotebookAuthorizationPath();
GsonBuilder builder = new GsonBuilder();
builder.setPrettyPrinting();
gson = builder.create();
try {
configStorage = ConfigStorage.getInstance(config);
loadFromFile();
} catch (IOException e) {
LOG.error("Error loading NotebookAuthorization", e);
@ -93,26 +91,10 @@ public class NotebookAuthorization {
}
private static void loadFromFile() throws IOException {
File settingFile = new File(filePath);
LOG.info(settingFile.getAbsolutePath());
if (!settingFile.exists()) {
// nothing to read
return;
NotebookAuthorizationInfoSaving info = configStorage.loadNotebookAuthorization();
if (info != null) {
authInfo = info.authInfo;
}
FileInputStream fis = new FileInputStream(settingFile);
InputStreamReader isr = new InputStreamReader(fis);
BufferedReader bufferedReader = new BufferedReader(isr);
StringBuilder sb = new StringBuilder();
String line;
while ((line = bufferedReader.readLine()) != null) {
sb.append(line);
}
isr.close();
fis.close();
String json = sb.toString();
NotebookAuthorizationInfoSaving info = NotebookAuthorizationInfoSaving.fromJson(json);
authInfo = info.authInfo;
}
public void setRoles(String user, Set<String> roles) {
@ -133,27 +115,14 @@ public class NotebookAuthorization {
}
private void saveToFile() {
String jsonString;
synchronized (authInfo) {
NotebookAuthorizationInfoSaving info = new NotebookAuthorizationInfoSaving();
info.authInfo = authInfo;
jsonString = gson.toJson(info);
}
try {
File settingFile = new File(filePath);
if (!settingFile.exists()) {
settingFile.createNewFile();
try {
configStorage.save(info);
} catch (IOException e) {
LOG.error("Error saving notebook authorization file", e);
}
FileOutputStream fos = new FileOutputStream(settingFile, false);
OutputStreamWriter out = new OutputStreamWriter(fos);
out.append(jsonString);
out.close();
fos.close();
} catch (IOException e) {
LOG.error("Error saving notebook authorization file: " + e.getMessage());
}
}

8
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.notebook;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -26,12 +27,10 @@ import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.security.SecureRandom;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
@ -182,6 +181,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
// strip white space from the beginning
this.text = newText;
this.dateUpdated = new Date();
parseText();
}
public void parseText() {
// parse text to get interpreter component
if (this.text != null) {
Matcher matcher = REPL_PATTERN.matcher(this.text);
@ -822,6 +825,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
return result1;
}
@Override
public String toJson() {
return Note.getGson().toJson(this);
}

78
zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/ConfigStorage.java Normal file
View file

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.storage;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.helium.HeliumConf;
import org.apache.zeppelin.interpreter.InterpreterInfoSaving;
import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving;
import org.apache.zeppelin.user.Credentials;
import org.apache.zeppelin.user.CredentialsInfoSaving;
import org.apache.zeppelin.util.ReflectionUtils;
import java.io.IOException;
/**
* Interface for storing zeppelin configuration.
*
* 1. interpreter-setting.json
* 2. helium.json
* 3. notebook-authorization.json
* 4. credentials.json
*
*/
public abstract class ConfigStorage {
private static ConfigStorage instance;
protected ZeppelinConfiguration zConf;
public static synchronized ConfigStorage getInstance(ZeppelinConfiguration zConf)
throws IOException {
if (instance == null) {
instance = createConfigStorage(zConf);
}
return instance;
}
private static ConfigStorage createConfigStorage(ZeppelinConfiguration zConf) throws IOException {
String configStorageClass =
zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS);
return ReflectionUtils.createClazzInstance(configStorageClass,
new Class[] {ZeppelinConfiguration.class}, new Object[] {zConf});
}
public ConfigStorage(ZeppelinConfiguration zConf) {
this.zConf = zConf;
}
public abstract void save(InterpreterInfoSaving settingInfos) throws IOException;
public abstract InterpreterInfoSaving loadInterpreterSettings() throws IOException;
public abstract void save(NotebookAuthorizationInfoSaving authorizationInfoSaving)
throws IOException;
public abstract NotebookAuthorizationInfoSaving loadNotebookAuthorization() throws IOException;
public abstract String loadCredentials() throws IOException;
public abstract void saveCredentials(String credentials) throws IOException;
}

122
zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.storage;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.hadoop.fs.Path;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.helium.HeliumConf;
import org.apache.zeppelin.interpreter.InterpreterInfoSaving;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.FileSystemStorage;
import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving;
import org.apache.zeppelin.user.CredentialsInfoSaving;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* It could be used either local file system or hadoop distributed file system,
* because FileSystem support both local file system and hdfs.
*
*/
public class FileSystemConfigStorage extends ConfigStorage {
private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemConfigStorage.class);
private FileSystemStorage fs;
private Path interpreterSettingPath;
private Path authorizationPath;
private Path credentialPath;
public FileSystemConfigStorage(ZeppelinConfiguration zConf) throws IOException {
super(zConf);
this.fs = FileSystemStorage.get(zConf);
this.fs.tryMkDir(new Path(zConf.getConfigFSDir()));
this.interpreterSettingPath = fs.makeQualified(new Path(zConf.getInterpreterSettingPath()));
this.authorizationPath = fs.makeQualified(new Path(zConf.getNotebookAuthorizationPath()));
this.credentialPath = fs.makeQualified(new Path(zConf.getCredentialsPath()));
}
@Override
public void save(InterpreterInfoSaving settingInfos) throws IOException {
LOGGER.info("Save Interpreter Settings to " + interpreterSettingPath);
fs.writeFile(settingInfos.toJson(), interpreterSettingPath, false);
}
@Override
public InterpreterInfoSaving loadInterpreterSettings() throws IOException {
if (!fs.exists(interpreterSettingPath)) {
LOGGER.warn("Interpreter Setting file {} is not existed", interpreterSettingPath);
return null;
}
LOGGER.info("Load Interpreter Setting from file: " + interpreterSettingPath);
String json = fs.readFile(interpreterSettingPath);
//TODO(zjffdu) This kind of post processing is ugly.
JsonParser jsonParser = new JsonParser();
JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json);
for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
// Always use separate interpreter process
// While we decided to turn this feature on always (without providing
// enable/disable option on GUI).
// previously created setting should turn this feature on here.
interpreterSetting.getOption();
interpreterSetting.convertPermissionsFromUsersToOwners(
jsonObject.getAsJsonObject("interpreterSettings")
.getAsJsonObject(interpreterSetting.getId()));
}
return infoSaving;
}
public void save(NotebookAuthorizationInfoSaving authorizationInfoSaving) throws IOException {
LOGGER.info("Save notebook authorization to file: " + authorizationPath);
fs.writeFile(authorizationInfoSaving.toJson(), authorizationPath, false);
}
@Override
public NotebookAuthorizationInfoSaving loadNotebookAuthorization() throws IOException {
if (!fs.exists(authorizationPath)) {
LOGGER.warn("Interpreter Setting file {} is not existed", authorizationPath);
return null;
}
LOGGER.info("Load notebook authorization from file: " + authorizationPath);
String json = this.fs.readFile(authorizationPath);
return NotebookAuthorizationInfoSaving.fromJson(json);
}
@Override
public String loadCredentials() throws IOException {
if (!fs.exists(credentialPath)) {
LOGGER.warn("Credential file {} is not existed", authorizationPath);
return null;
}
LOGGER.info("Load Credential from file: " + authorizationPath);
return this.fs.readFile(credentialPath);
}
@Override
public void saveCredentials(String credentials) throws IOException {
LOGGER.info("Save Credentials to file: " + credentialPath);
fs.writeFile(credentials, credentialPath, false);
}
}

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Credentials.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Credentials.java
View file

@ -20,18 +20,22 @@ package org.apache.zeppelin.user;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.zeppelin.common.JsonSerializable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.nio.file.attribute.PosixFilePermission;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
@ -48,7 +52,7 @@ public class Credentials {
File credentialsFile;
private Encryptor encryptor;
/**
* Wrapper fro user credentials. It can load credentials from a file if credentialsPath is
* supplied, and will encrypt the file if an encryptKey is supplied.

0
zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/CredentialsInfoSaving.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/user/CredentialsInfoSaving.java
View file

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Encryptor.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Encryptor.java
View file

@ -17,11 +17,6 @@
package org.apache.zeppelin.user;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Arrays;
import org.bouncycastle.crypto.BufferedBlockCipher;
import org.bouncycastle.crypto.InvalidCipherTextException;
import org.bouncycastle.crypto.engines.AESEngine;
@ -30,6 +25,8 @@ import org.bouncycastle.crypto.paddings.ZeroBytePadding;
import org.bouncycastle.crypto.params.KeyParameter;
import org.bouncycastle.util.encoders.Base64;
import java.io.IOException;
/**
* Encrypt/decrypt arrays of bytes!
*/

120
zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
View file

@ -17,91 +17,87 @@
package org.apache.zeppelin.conf;
import junit.framework.Assert;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.net.MalformedURLException;
import java.util.List;
/**
* Created by joelz on 8/19/15.
*/
public class ZeppelinConfigurationTest {
@Before
public void clearSystemVariables() {
System.clearProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName());
}
@Before
public void clearSystemVariables() {
System.clearProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName());
}
@Test
public void getAllowedOrigins2Test() throws MalformedURLException, ConfigurationException {
@Test
public void getAllowedOrigins2Test() throws MalformedURLException, ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site2.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(2, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
Assert.assertEquals("http://otherhost.com", origins.get(1));
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site2.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(2, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
Assert.assertEquals("http://otherhost.com", origins.get(1));
}
@Test
public void getAllowedOrigins1Test() throws MalformedURLException, ConfigurationException {
@Test
public void getAllowedOrigins1Test() throws MalformedURLException, ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site1.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site1.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
}
@Test
public void getAllowedOriginsNoneTest() throws MalformedURLException, ConfigurationException {
@Test
public void getAllowedOriginsNoneTest() throws MalformedURLException, ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
}
@Test
public void isWindowsPathTestTrue() throws ConfigurationException {
@Test
public void isWindowsPathTestTrue() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
Boolean isIt = conf.isWindowsPath("c:\\test\\file.txt");
Assert.assertTrue(isIt);
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
Boolean isIt = conf.isWindowsPath("c:\\test\\file.txt");
Assert.assertTrue(isIt);
}
@Test
public void isWindowsPathTestFalse() throws ConfigurationException {
@Test
public void isWindowsPathTestFalse() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
Boolean isIt = conf.isWindowsPath("~/test/file.xml");
Assert.assertFalse(isIt);
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
Boolean isIt = conf.isWindowsPath("~/test/file.xml");
Assert.assertFalse(isIt);
}
@Test
public void getNotebookDirTest() throws ConfigurationException {
@Test
public void getNotebookDirTest() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
String notebookLocation = conf.getNotebookDir();
Assert.assertEquals("notebook", notebookLocation);
}
@Test
public void isNotebookPublicTest() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
boolean isIt = conf.isNotebookPublic();
assertTrue(isIt);
}
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
String notebookLocation = conf.getNotebookDir();
Assert.assertEquals("notebook", notebookLocation);
}
@Test
public void isRequestHeaderSizeDefaultValueCorrect() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
assertEquals((Integer)8192, conf.getJettyRequestHeaderSize());
}
@Test
public void isNotebookPublicTest() throws ConfigurationException {
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
boolean isIt = conf.isNotebookPublic();
assertTrue(isIt);
}
@Test
public void getPathTest() throws ConfigurationException {
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"));
Assert.assertEquals("/usr/lib/zeppelin", conf.getZeppelinHome());
Assert.assertEquals("/usr/lib/zeppelin/conf", conf.getConfDir());
}
}

1
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -39,6 +39,7 @@ public class InterpreterFactoryTest extends AbstractInterpreterTest {
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "");
// EchoInterpreter is the default interpreter because mock1 is the default interpreter group
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "test") instanceof RemoteInterpreter);

4
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
View file

@ -38,7 +38,8 @@ public class ShellScriptLauncherTest {
properties.setProperty("ENV_1", "VALUE_1");
properties.setProperty("property_1", "value_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "groupName", "name");
option.setUserImpersonate(true);
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
@ -48,6 +49,7 @@ public class ShellScriptLauncherTest {
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(1, interpreterProcess.getEnv().size());
assertEquals("VALUE_1", interpreterProcess.getEnv().get("ENV_1"));
assertEquals(true, interpreterProcess.isUserImpersonated());
}
}

33
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
View file

@ -42,13 +42,13 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "spark", "spark");
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(2, interpreterProcess.getEnv().size());
assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
@ -67,13 +67,13 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "spark", "spark");
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(2, interpreterProcess.getEnv().size());
assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
@ -93,13 +93,13 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "spark", "spark");
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(2, interpreterProcess.getEnv().size());
assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
@ -118,13 +118,13 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "spark", "spark");
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(3, interpreterProcess.getEnv().size());
assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
@ -145,17 +145,18 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "intpGroupId", "groupId", "spark", "spark");
option.setUserImpersonate(true);
InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertTrue(interpreterProcess.getInterpreterDir().endsWith("/interpreter/spark"));
assertTrue(interpreterProcess.getLocalRepoDir().endsWith("/local-repo/groupId"));
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(3, interpreterProcess.getEnv().size());
assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME"));
assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true --proxy-user user1", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF"));
}
}

40
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -361,6 +361,46 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testScheduleAgainstRunningAndPendingParagraph() throws InterruptedException, IOException {
// create a note
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreterBinding("user", note.getId(),
interpreterSettingManager.getInterpreterSettingIds());
// append running and pending paragraphs to the note
for (Status status: new Status[]{Status.RUNNING, Status.PENDING}) {
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
p.setConfig(config);
p.setText("p");
p.setStatus(status);
assertNull(p.getDateFinished());
}
// set cron scheduler, once a second
Map config = note.getConfig();
config.put("enabled", true);
config.put("cron", "* * * * * ?");
note.setConfig(config);
notebook.refreshCron(note.getId());
Thread.sleep(2 * 1000);
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.getId());
Thread.sleep(2 * 1000);
// check if the executions of the running and pending paragraphs were skipped
for (Paragraph p : note.paragraphs) {
assertNull(p.getDateFinished());
}
// remove the note
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testSchedulePoolUsage() throws InterruptedException, IOException {
final int timeout = 30;

4
zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/CredentialsTest.java → zeppelin-zengine/src/test/java/org/apache/zeppelin/user/CredentialsTest.java
View file

@ -17,12 +17,12 @@
package org.apache.zeppelin.user;
import static org.junit.Assert.*;
import org.junit.Test;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
public class CredentialsTest {
@Test

0
zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/EncryptorTest.java → zeppelin-zengine/src/test/java/org/apache/zeppelin/user/EncryptorTest.java
View file