Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' of https://github.com/apache/zeppelin into java1.8doc

Browse source
This commit is contained in:
Maxim Belousov 2018-07-03 19:49:48 +03:00
commit c91fa47855
1105 changed files with 111951 additions and 36911 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
.appveyor.yml Normal file
View file

@ -0,0 +1,33 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
version: '1.0.0-dev.{build}'
shallow_clone: true
build: off
os:
- Visual Studio 2015
install:
- echo "Install"
build_script:
- echo "Build"

4
.github/PULL_REQUEST_TEMPLATE vendored
View file

@ -14,7 +14,9 @@ First time? Check out the contributing guide - https://zeppelin.apache.org/contr
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
* First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
### Screenshots (if appropriate)

9
.gitignore vendored
View file

@ -18,6 +18,9 @@ spark-1.*-bin-hadoop*
lens/lens-cli-hist.log
# Zeppelin server
zeppelin-server/local-repo
zeppelin-server/src/main/resources/zeppelin-site.xml
# conf file
conf/zeppelin-env.sh
@ -100,6 +103,9 @@ Thumbs.db
target/
**/target/
# maven flattened pom files
**/.flattened-pom.xml
# Generated by Jekyll
docs/_site/
@ -119,3 +125,6 @@ tramp
# tmp files
/tmp/
# Git properties
**/git.properties

138
.travis.yml
View file

@ -30,68 +30,133 @@ cache:
addons:
apt:
sources:
- r-packages-precise
- r-packages-trusty
packages:
- r-base-dev
env:
global:
# Interpreters does not required by zeppelin-server integration tests
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy,!sap'
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
- jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="false" SCALA_VER="2.11" PROFILE="-Prat -Pr" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.1.0 and scala 2.11
# Run e2e tests (in zeppelin-web)
# chrome dropped the support for precise (ubuntu 12.04), so need to use trusty
# also, can't use JDK 7 in trusty: https://github.com/travis-ci/travis-ci/issues/7884
- os: linux
sudo: false
dist: trusty
jdk: "oraclejdk8"
env: BUILD_PLUGINS="false" CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
addons:
apt:
packages:
- google-chrome-stable
# Test core modules
# Several tests were excluded from this configuration due to the following issues:
# HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
# After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
- sudo: required
jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with spark 2.0.2 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.6.3 with scala 2.10
- sudo: required
jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10 -Plivy-0.2" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11 -Dscala.version=2.11.7 -Dscala.binary.version=2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="true" PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
- jdk: "oraclejdk8"
dist: trusty
addons:
firefox: "31.0"
env: BUILD_PLUGINS="true" CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phelium-dev -Pexamples -Pintegration -Pscala-2.10" BUILD_FLAG="install -DskipTests -DskipRat -pl ${INTERPRETERS}" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-integration -DfailIfNoTests=false"
# Test python/pyspark with python 2
- jdk: "oraclejdk7"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
# Test interpreter modules
- jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
# Test python/pyspark with python 3
- jdk: "oraclejdk7"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
# Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately
# Integration test of spark interpreter with different spark versions under python2, only run ZeppelinSparkClusterTest. Also run spark unit test of spark 2.2 in this build.
- sudo: required
jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="true" PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Integration test of spark interpreter with different spark versions under python3, only run SparkIntegrationTest. Also run spark unit test of spark 1.6 in this build.
- sudo: required
jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pspark-1.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 2.1.0 with scala 2.11
- jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false"
# Test spark module for 2.0.2 with scala 2.11
- jdk: "oraclejdk8"
dist: trusty
env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11 -Pr" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies,r" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.rinterpreter.*,org.apache.spark.api.r.* -DfailIfNoTests=false"
# Test python/pyspark with python 2, livy 0.5
- sudo: required
dist: trusty
jdk: "oraclejdk8"
env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
# Test livy 0.5 with spark 2.2.0 under python3
- sudo: required
dist: trusty
jdk: "openjdk8"
env: BUILD_PLUGINS="false" PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
before_install:
# check files included in commit range, clear bower_components if a bower.json file has changed.
# bower cache clearing can also be forced by putting "bower clear" or "clear bower" in a commit message
- changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE 2>/dev/null) || changedfiles=""
- echo $changedfiles
- hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true);
- gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog=""
- clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
- if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache verify; else echo "Using cached bower_components."; fi
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
- ./testing/install_external_dependencies.sh
- ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
- ./dev/change_scala_version.sh $SCALA_VER
#- ./dev/change_scala_version.sh $SCALA_VER
- source ~/.environ
install:
- echo "mvn $BUILD_FLAG $MODULES $PROFILE -B"
- mvn $BUILD_FLAG $MODULES $PROFILE -B
- if [ $BUILD_PLUGINS == "true" ]; then echo "mvn clean package -pl zeppelin-plugins -amd -B"; mvn clean package -pl zeppelin-plugins -amd -B; fi
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
- if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-$LIVY_VER-bin; fi
- if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
- if [[ -n $SPARK_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
- if [[ -n $SPARK_VER ]]; then echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh; fi
- echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh
- echo "export SPARK_PRINT_LAUNCH_COMMAND=true" >> conf/zeppelin-env.sh
- export SPARK_PRINT_LAUNCH_COMMAND=true
- tail conf/zeppelin-env.sh
# https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
- if [[ -n $TEST_MODULES ]]; then export DISPLAY=:99.0; sh -e /etc/init.d/xvfb start; sleep 3; fi
# display info log for debugging
- if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
script:
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
- echo "mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
- mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS
after_success:
@ -100,15 +165,14 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
- cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
- cat spark-*/logs/*
- cat livy/target/tmp/*/output.log
- ls -R livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
- cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
- cat livy/target/tmp/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
- cat livy/target/tmp/livy-int-test/*/output.log
- ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stdout
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/com.cloudera.livy.test.framework.MiniYarnMain/*/*/*/stderr
- ls -R livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stdout
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stderr
- cat zeppelin-zengine/target/org.apache.zeppelin.interpreter.MiniHadoopCluster/*/*/*/stdout

18
LICENSE
View file

@ -235,6 +235,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) Simple line icons v1.0.0 (http://thesabbir.github.io/simple-line-icons/) - https://github.com/thesabbir/simple-line-icons/tree/1.0.0
(The MIT License) jekyll-bootstrap 0.3.0 (https://github.com/plusjade/jekyll-bootstrap) - https://github.com/plusjade/jekyll-bootstrap
(The MIT License) jekyll 1.3.0 (http://jekyllrb.com/) - https://github.com/jekyll/jekyll/blob/v1.3.0/LICENSE
(The MIT License) ngInfiniteScroll 1.3.4 (https://github.com/sroze/ngInfiniteScroll) - https://github.com/sroze/ngInfiniteScroll/blob/master/LICENSE
========================================================================
MIT-style licenses
@ -255,6 +256,10 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(Apache 2.0) Bootstrap v3.0.2 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.0.2/LICENSE
(Apache 2.0) Software under ./bigquery/* was developed at Google (http://www.google.com/). Licensed under the Apache v2.0 License.
(Apache 2.0) Roboto Font (https://github.com/google/roboto/)
(Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras)
(Apache 2.0) Nimbus JOSE+JWT (https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home)
(Apache 2.0) jarchivelib (https://github.com/thrau/jarchivelib)
(Apache 2.0) Google Cloud Client Library for Java (https://github.com/GoogleCloudPlatform/google-cloud-java)
========================================================================
BSD 3-Clause licenses
@ -268,10 +273,21 @@ The following components are provided under the BSD 3-Clause license. See file
(BSD 3 Clause) portions of Scala (http://www.scala-lang.org/download) - http://www.scala-lang.org/download/#License
r/src/main/scala/scala/Console.scala
(BSD 3 Clause) diff.js (https://github.com/kpdecker/jsdiff)
(BSD 3-Clause) Google Auth Library for Java (https://github.com/google/google-auth-library-java)
========================================================================
BSD 2-Clause licenses
========================================================================
The following components are provided under the BSD 3-Clause license. See file headers and project links for details.
The following components are provided under the BSD 2-Clause license. See file headers and project links for details.
(BSD 2 Clause) portions of SQLLine (http://sqlline.sourceforge.net/) - http://sqlline.sourceforge.net/#license
jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
========================================================================
Jython Software License
========================================================================
The following components are provided under the Jython Software License. See file headers and project links for details.
(Jython Software License) jython-standalone - http://www.jython.org/

10
README.md
View file

@ -1,7 +1,7 @@
# Apache Zeppelin
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
**Documentation:** [User Guide](https://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](https://zeppelin.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://travis-ci.org/apache/zeppelin.svg?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
@ -15,15 +15,15 @@ Core feature:
* Built-in Apache Spark support
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org)
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](https://zeppelin.apache.org)
## Getting Started
### Install binary package
Please go to [install](http://zeppelin.apache.org/docs/snapshot/install/install.html) to install Apache Zeppelin from binary package.
Please go to [install](https://zeppelin.apache.org/docs/latest/install/install.html) to install Apache Zeppelin from binary package.
### Build from source
Please check [Build from source](http://zeppelin.apache.org/docs/snapshot/install/build.html) to build Zeppelin from source.
Please check [Build from source](https://zeppelin.apache.org/docs/latest/install/build.html) to build Zeppelin from source.

433
_tools/checkstyle.xml
View file

@ -15,268 +15,275 @@ limitations under the License.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!-- This is a checkstyle configuration file. For descriptions of what the
following rules do, please see the checkstyle configuration page at http://checkstyle.sourceforge.net/config.html -->
<module name="Checker">
<module name="FileTabCharacter">
<!-- Checks that there are no tab characters in the file. -->
</module>
<module name="FileTabCharacter">
<!-- Checks that there are no tab characters in the file. -->
</module>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf" />
</module>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpSingleline">
<!-- Checks that FIXME is not used in comments. TODO is preferred. -->
<property name="format" value="((//.*)|(\*.*))FIXME" />
<property name="message"
value='TODO is preferred to FIXME. e.g. "TODO(johndoe): Refactor when v2 is released."' />
</module>
<module name="RegexpSingleline">
<!-- Checks that FIXME is not used in comments. TODO is preferred. -->
<property name="format" value="((//.*)|(\*.*))FIXME"/>
<property name="message"
value='TODO is preferred to FIXME. e.g. "TODO(johndoe): Refactor when v2 is released."'/>
</module>
<module name="RegexpSingleline">
<!-- Checks that TODOs are named. (Actually, just that they are followed
by an open paren.) -->
<property name="format" value="((//.*)|(\*.*))TODO[^(]" />
<property name="message"
value='All TODOs should be named. e.g. "TODO(johndoe): Refactor when v2 is released."' />
</module>
<module name="RegexpSingleline">
<!-- Checks that TODOs are named. (Actually, just that they are followed
by an open paren.) -->
<property name="format" value="((//.*)|(\*.*))TODO[^(]"/>
<property name="message"
value='All TODOs should be named. e.g. "TODO(johndoe): Refactor when v2 is released."'/>
</module>
<!-- <module name="JavadocPackage"> - Checks that each Java package has
a Javadoc file used for commenting. Only allows a package-info.java, not
package.html. </module> -->
<!-- All Java AST specific tests live under TreeWalker module. -->
<module name="TreeWalker">
<!-- <module name="JavadocPackage"> - Checks that each Java package has
a Javadoc file used for commenting. Only allows a package-info.java, not
package.html. </module> -->
<!-- All Java AST specific tests live under TreeWalker module. -->
<module name="TreeWalker">
<!-- IMPORT CHECKS -->
<!-- IMPORT CHECKS -->
<module name="RedundantImport">
<!-- Checks for redundant import statements. -->
<property name="severity" value="error" />
</module>
<!-- <module name="ImportOrder"> Checks for out of order import statements
<property name="severity" value="warning"/> <property name="groups" value="com.google,android,junit,net,org,java,javax"/>
This ensures that static imports go first <property name="option" value="top"/>
<property name="tokens" value="STATIC_IMPORT, IMPORT"/> </module> -->
<!-- JAVADOC CHECKS -->
<module name="RedundantImport">
<!-- Checks for redundant import statements. -->
<property name="severity" value="error"/>
</module>
<module name="ImportOrder">
<property name="severity" value="warning"/>
<property name="groups" value="com.google,junit,net,org,java,javax,*,org.apache.zeppelin"/>
<property name="option" value="top"/>
<property name="tokens" value="STATIC_IMPORT, IMPORT"/>
</module>
<!-- JAVADOC CHECKS -->
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocMethod">
<property name="scope" value="protected" />
<property name="severity" value="warning" />
<property name="allowMissingJavadoc" value="true" />
<property name="allowMissingParamTags" value="true" />
<property name="allowMissingReturnTag" value="true" />
<property name="allowMissingThrowsTags" value="true" />
<property name="allowThrowsTagsForSubclasses" value="true" />
<property name="allowUndeclaredRTE" value="true" />
</module>
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocMethod">
<property name="scope" value="protected"/>
<property name="severity" value="warning"/>
<property name="allowMissingJavadoc" value="true"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="allowMissingThrowsTags" value="true"/>
<property name="allowThrowsTagsForSubclasses" value="true"/>
<property name="allowUndeclaredRTE" value="true"/>
</module>
<module name="JavadocType">
<property name="scope" value="protected" />
<property name="severity" value="error" />
</module>
<module name="JavadocStyle">
<property name="severity" value="warning"/>
</module>
<module name="JavadocStyle">
<property name="severity" value="warning" />
</module>
<!-- NAMING CHECKS -->
<!-- NAMING CHECKS -->
<!-- Item 38 - Adhere to generally accepted naming conventions -->
<!-- Item 38 - Adhere to generally accepted naming conventions -->
<module name="PackageName">
<!-- Validates identifiers for package names against the supplied expression. -->
<!-- Here the default checkstyle rule restricts package name parts to
seven characters, this is not in line with common practice at Google. -->
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$"/>
<property name="severity" value="warning"/>
</module>
<module name="PackageName">
<!-- Validates identifiers for package names against the supplied expression. -->
<!-- Here the default checkstyle rule restricts package name parts to
seven characters, this is not in line with common practice at Google. -->
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$" />
<property name="severity" value="warning" />
</module>
<module name="TypeNameCheck">
<!-- Validates static, final fields against the expression "^[A-Z][a-zA-Z0-9]*$". -->
<metadata name="altname" value="TypeName"/>
<property name="severity" value="warning"/>
</module>
<module name="TypeNameCheck">
<!-- Validates static, final fields against the expression "^[A-Z][a-zA-Z0-9]*$". -->
<metadata name="altname" value="TypeName" />
<property name="severity" value="warning" />
</module>
<module name="ConstantNameCheck">
<!-- Validates non-private, static, final fields against the supplied
public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". -->
<metadata name="altname" value="ConstantName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="false"/>
<property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$"/>
<message key="name.invalidPattern"
value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)."/>
<property name="severity" value="warning"/>
</module>
<module name="ConstantNameCheck">
<!-- Validates non-private, static, final fields against the supplied
public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". -->
<metadata name="altname" value="ConstantName" />
<property name="applyToPublic" value="true" />
<property name="applyToProtected" value="true" />
<property name="applyToPackage" value="true" />
<property name="applyToPrivate" value="false" />
<property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$" />
<message key="name.invalidPattern"
value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)." />
<property name="severity" value="warning" />
</module>
<module name="StaticVariableNameCheck">
<!-- Validates static, non-final fields against the supplied expression
"^[a-z][a-zA-Z0-9]*_?$". -->
<metadata name="altname" value="StaticVariableName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>
<property name="severity" value="warning"/>
</module>
<module name="StaticVariableNameCheck">
<!-- Validates static, non-final fields against the supplied expression
"^[a-z][a-zA-Z0-9]*_?$". -->
<metadata name="altname" value="StaticVariableName" />
<property name="applyToPublic" value="true" />
<property name="applyToProtected" value="true" />
<property name="applyToPackage" value="true" />
<property name="applyToPrivate" value="true" />
<property name="format" value="^[a-z][a-zA-Z0-9]*_?$" />
<property name="severity" value="warning" />
</module>
<module name="MemberNameCheck">
<!-- Validates non-static members against the supplied expression. -->
<metadata name="altname" value="MemberName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*$"/>
<property name="severity" value="warning"/>
</module>
<module name="MemberNameCheck">
<!-- Validates non-static members against the supplied expression. -->
<metadata name="altname" value="MemberName" />
<property name="applyToPublic" value="true" />
<property name="applyToProtected" value="true" />
<property name="applyToPackage" value="true" />
<property name="applyToPrivate" value="true" />
<property name="format" value="^[a-z][a-zA-Z0-9]*$" />
<property name="severity" value="warning" />
</module>
<module name="MethodNameCheck">
<!-- Validates identifiers for method names. -->
<metadata name="altname" value="MethodName"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$"/>
<property name="severity" value="warning"/>
</module>
<module name="MethodNameCheck">
<!-- Validates identifiers for method names. -->
<metadata name="altname" value="MethodName" />
<property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$" />
<property name="severity" value="warning" />
</module>
<module name="ParameterName">
<!-- Validates identifiers for method parameters against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="ParameterName">
<!-- Validates identifiers for method parameters against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning" />
</module>
<module name="LocalFinalVariableName">
<!-- Validates identifiers for local final variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="LocalFinalVariableName">
<!-- Validates identifiers for local final variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning" />
</module>
<module name="LocalVariableName">
<!-- Validates identifiers for local variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning" />
</module>
<module name="LocalVariableName">
<!-- Validates identifiers for local variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<!-- LENGTH and CODING CHECKS -->
<!-- LENGTH and CODING CHECKS -->
<module name="LineLength">
<!-- Checks if a line is too long. -->
<property name="max"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.max}"
default="100" />
<property name="severity" value="error" />
<module name="LineLength">
<!-- Checks if a line is too long. -->
<property name="max"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.max}"
default="100"/>
<property name="severity" value="error"/>
<!-- The default ignore pattern exempts the following elements: - import
statements - long URLs inside comments -->
<!-- The default ignore pattern exempts the following elements: - import
statements - long URLs inside comments -->
<property name="ignorePattern"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$" />
</module>
<property name="ignorePattern"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/>
</module>
<module name="LeftCurly">
<!-- Checks for placement of the left curly brace ('{'). -->
<property name="severity" value="warning" />
</module>
<module name="LeftCurly">
<!-- Checks for placement of the left curly brace ('{'). -->
<property name="severity" value="warning"/>
</module>
<module name="RightCurly">
<!-- Checks right curlies on CATCH, ELSE, and TRY blocks are on the same
line. e.g., the following example is fine: <pre> if { ... } else </pre> -->
<!-- This next example is not fine: <pre> if { ... } else </pre> -->
<property name="option" value="same" />
<property name="severity" value="warning" />
</module>
<module name="RightCurly">
<!-- Checks right curlies on CATCH, ELSE, and TRY blocks are on the same
line. e.g., the following example is fine: <pre> if { ... } else </pre> -->
<!-- This next example is not fine: <pre> if { ... } else </pre> -->
<property name="option" value="same"/>
<property name="severity" value="warning"/>
</module>
<!-- Checks for braces around if and else blocks -->
<module name="NeedBraces">
<property name="severity" value="warning" />
<property name="tokens"
value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO" />
</module>
<!-- Checks for braces around if and else blocks -->
<module name="NeedBraces">
<property name="severity" value="warning"/>
<property name="tokens"
value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/>
</module>
<module name="UpperEll">
<!-- Checks that long constants are defined with an upper ell. -->
<property name="severity" value="error" />
</module>
<module name="UpperEll">
<!-- Checks that long constants are defined with an upper ell. -->
<property name="severity" value="error"/>
</module>
<module name="FallThrough">
<!-- Warn about falling through to the next case statement. Similar to
javac -Xlint:fallthrough, but the check is suppressed if a single-line comment
on the last non-blank line preceding the fallen-into case contains 'fall
through' (or some other variants which we don't publicized to promote consistency). -->
<property name="reliefPattern"
value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on" />
<property name="severity" value="error" />
</module>
<module name="FallThrough">
<!-- Warn about falling through to the next case statement. Similar to
javac -Xlint:fallthrough, but the check is suppressed if a single-line comment
on the last non-blank line preceding the fallen-into case contains 'fall
through' (or some other variants which we don't publicized to promote consistency). -->
<property name="reliefPattern"
value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/>
<property name="severity" value="error"/>
</module>
<!-- MODIFIERS CHECKS -->
<!-- MODIFIERS CHECKS -->
<module name="ModifierOrder">
<!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and
8.4.3. The prescribed order is: public, protected, private, abstract, static,
final, transient, volatile, synchronized, native, strictfp -->
</module>
<module name="ModifierOrder">
<!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and
8.4.3. The prescribed order is: public, protected, private, abstract, static,
final, transient, volatile, synchronized, native, strictfp -->
</module>
<!-- WHITESPACE CHECKS -->
<!-- WHITESPACE CHECKS -->
<module name="WhitespaceAround">
<!-- Checks that various tokens are surrounded by whitespace. This includes
most binary operators and keywords followed by regular or curly braces. -->
<property name="tokens"
value="ASSIGN, BAND, BAND_ASSIGN, BOR,
<module name="WhitespaceAround">
<!-- Checks that various tokens are surrounded by whitespace. This includes
most binary operators and keywords followed by regular or curly braces. -->
<property name="tokens"
value="ASSIGN, BAND, BAND_ASSIGN, BOR,
BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,
EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,
MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,
SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN" />
<property name="severity" value="error" />
</module>
SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>
<property name="severity" value="error"/>
</module>
<module name="WhitespaceAfter">
<!-- Checks that commas, semicolons and typecasts are followed by whitespace. -->
<property name="tokens" value="COMMA, SEMI, TYPECAST" />
</module>
<module name="WhitespaceAfter">
<!-- Checks that commas, semicolons and typecasts are followed by whitespace. -->
<property name="tokens" value="COMMA, SEMI, TYPECAST"/>
</module>
<module name="NoWhitespaceAfter">
<!-- Checks that there is no whitespace after various unary operators.
Linebreaks are allowed. -->
<property name="tokens"
value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS,
UNARY_PLUS" />
<property name="allowLineBreaks" value="true" />
<property name="severity" value="error" />
</module>
<module name="NoWhitespaceAfter">
<!-- Checks that there is no whitespace after various unary operators.
Linebreaks are allowed. -->
<property name="tokens"
value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS,
UNARY_PLUS"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="NoWhitespaceBefore">
<!-- Checks that there is no whitespace before various unary operators.
Linebreaks are allowed. -->
<property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC" />
<property name="allowLineBreaks" value="true" />
<property name="severity" value="error" />
</module>
<module name="NoWhitespaceBefore">
<!-- Checks that there is no whitespace before various unary operators.
Linebreaks are allowed. -->
<property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="ParenPad">
<!-- Checks that there is no whitespace before close parens or after open
parens. -->
<property name="severity" value="warning" />
</module>
<module name="ParenPad">
<!-- Checks that there is no whitespace before close parens or after open
parens. -->
<property name="severity" value="warning"/>
</module>
<module name="Indentation">
<!-- Checks code indentation -->
<property name="basicOffset" value="2" />
</module>
</module>
<module name="Indentation">
<!-- Checks code indentation -->
<property name="basicOffset" value="2"/>
<property name="caseIndent" value="2"/>
</module>
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
<module name="UnusedImports"/>
<module name="RedundantImport"/>
<module name="RedundantModifier"/>
<module name="AvoidStarImport"/>
</module>
</module>

65
alluxio/pom.xml
View file

@ -20,20 +20,21 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-alluxio</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Alluxio interpreter</name>
<properties>
<alluxio.version>1.0.0</alluxio.version>
<interpreter.name>alluxio</interpreter.name>
</properties>
<dependencies>
@ -47,6 +48,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>15.0</version>
</dependency>
<dependency>
@ -128,54 +130,19 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

34
alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java
View file

@ -18,23 +18,28 @@
package org.apache.zeppelin.alluxio;
import java.io.IOException;
import java.io.PrintStream;
import java.io.ByteArrayOutputStream;
import java.util.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import alluxio.Configuration;
import alluxio.shell.AlluxioShell;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
/**
* Alluxio interpreter for Zeppelin.
*/
@ -71,7 +76,7 @@ public class AlluxioInterpreter extends Interpreter {
@Override
public void open() {
logger.info("Starting Alluxio shell to connect to " + alluxioMasterHostname +
" on port " + alluxioMasterPort);
" on port " + alluxioMasterPort);
System.setProperty(ALLUXIO_MASTER_HOSTNAME, alluxioMasterHostname);
System.setProperty(ALLUXIO_MASTER_PORT, alluxioMasterPort);
@ -166,7 +171,8 @@ public class AlluxioInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
String lastWord = "";
if (words.length > 0) {
@ -176,7 +182,7 @@ public class AlluxioInterpreter extends Interpreter {
List<InterpreterCompletion> voices = new LinkedList<>();
for (String command : keywords) {
if (command.startsWith(lastWord)) {
voices.add(new InterpreterCompletion(command, command));
voices.add(new InterpreterCompletion(command, command, CompletionType.command.name()));
}
}
return voices;

9
alluxio/src/main/resources/interpreter-setting.json
View file

@ -8,17 +8,20 @@
"envName": "ALLUXIO_MASTER_HOSTNAME",
"propertyName": "alluxio.master.hostname",
"defaultValue": "localhost",
"description": "Alluxio master hostname"
"description": "Alluxio master hostname",
"type": "string"
},
"alluxio.master.port": {
"envName": "ALLUXIO_MASTER_PORT",
"propertyName": "alluxio.master.port",
"defaultValue": "19998",
"description": "Alluxio master port"
"description": "Alluxio master port",
"type": "number"
}
},
"editor": {
"editOnDblClick": false
"editOnDblClick": false,
"completionSupport": true
}
}
]

79
alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java
View file

@ -18,6 +18,11 @@
package org.apache.zeppelin.alluxio;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -27,26 +32,25 @@ import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import alluxio.client.WriteType;
import alluxio.client.file.URIStatus;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.*;
import alluxio.Constants;
import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.client.FileSystemTestUtils;
import alluxio.client.WriteType;
import alluxio.client.file.FileInStream;
import alluxio.client.file.FileSystem;
import alluxio.exception.ExceptionMessage;
import alluxio.client.file.URIStatus;
import alluxio.exception.AlluxioException;
import alluxio.exception.ExceptionMessage;
import alluxio.master.LocalAlluxioCluster;
import alluxio.shell.command.CommandUtils;
import alluxio.util.FormatUtils;
import alluxio.util.io.BufferUtils;
import alluxio.util.io.PathUtils;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
public class AlluxioInterpreterTest {
private AlluxioInterpreter alluxioInterpreter;
@ -78,28 +82,32 @@ public class AlluxioInterpreterTest {
@Test
public void testCompletion() {
List expectedResultOne = Arrays.asList(
new InterpreterCompletion("cat", "cat"),
new InterpreterCompletion("chgrp", "chgrp"),
new InterpreterCompletion("chmod", "chmod"),
new InterpreterCompletion("chown", "chown"),
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
new InterpreterCompletion("copyToLocal", "copyToLocal"),
new InterpreterCompletion("count", "count"),
new InterpreterCompletion("createLineage", "createLineage"));
new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()),
new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name()));
List expectedResultTwo = Arrays.asList(
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
new InterpreterCompletion("copyToLocal", "copyToLocal"),
new InterpreterCompletion("count", "count"));
new InterpreterCompletion("copyFromLocal", "copyFromLocal",
CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal",
CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()));
List expectedResultThree = Arrays.asList(
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
new InterpreterCompletion("copyToLocal", "copyToLocal"));
new InterpreterCompletion("copyFromLocal", "copyFromLocal",
CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal",
CompletionType.command.name()));
List expectedResultNone = new ArrayList<>();
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);
List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0);
List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0);
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0, null);
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0, null);
List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0, null);
List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0, null);
List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0, null);
Assert.assertEquals(expectedResultOne, resultOne);
Assert.assertEquals(expectedResultTwo, resultTwo);
@ -141,7 +149,8 @@ public class AlluxioInterpreterTest {
Assert.assertEquals(Code.SUCCESS, output.code());
Assert.assertArrayEquals(expected,
output.message().get(0).getData().substring(0, output.message().get(0).getData().length() - 1).getBytes());
output.message().get(0).getData().substring(0,
output.message().get(0).getData().length() - 1).getBytes());
}
@Test
@ -186,8 +195,10 @@ public class AlluxioInterpreterTest {
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.CACHE_THROUGH, 10, 10);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileB", WriteType.MUST_CACHE, 10, 10);
int memPercentageA = fs.getStatus(new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage();
int memPercentageB = fs.getStatus(new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage();
int memPercentageA = fs.getStatus(
new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage();
int memPercentageB = fs.getStatus(
new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage();
Assert.assertFalse(memPercentageA == 0);
Assert.assertTrue(memPercentageB == 100);
@ -357,7 +368,8 @@ public class AlluxioInterpreterTest {
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected += String.format(format, FormatUtils.getSizeFromBytes(10),
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory", "/testRoot/testFileA");
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory",
"/testRoot/testFileA");
expected += String.format(format, FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
expected += String.format(format, FormatUtils.getSizeFromBytes(30),
@ -395,7 +407,8 @@ public class AlluxioInterpreterTest {
"/testRoot/testFileA");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "",
"/testRoot/testDir");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(20),
CommandUtils.convertMsToDate(files[2].getCreationTimeMs()), "In Memory",
@ -476,4 +489,4 @@ public class AlluxioInterpreterTest {
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(size, read));
}
}
}

67
angular/pom.xml
View file

@ -20,18 +20,22 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-angular</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Angular interpreter</name>
<properties>
<interpreter.name>angular</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@ -61,54 +65,19 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

3
angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java
View file

@ -67,7 +67,8 @@ public class AngularInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return new LinkedList<>();
}

3
angular/src/main/resources/interpreter-setting.json
View file

@ -6,7 +6,8 @@
"properties": {
},
"editor": {
"editOnDblClick": true
"editOnDblClick": true,
"completionSupport": false
}
}
]

2
beam/README.md
View file

@ -8,7 +8,7 @@ Current interpreter implementation supports the static repl. It compiles the cod
You have to first build the Beam interpreter by enable the **beam** profile as follows:
```
mvn clean package -Pbeam -DskipTests
mvn clean package -Pbeam -DskipTests -Pscala-2.10
```
### Notice

85
beam/pom.xml
View file

@ -20,27 +20,28 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-beam</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Beam interpreter</name>
<properties>
<beam.hadoop.version>2.3.0</beam.hadoop.version>
<beam.spark.version>1.6.2</beam.spark.version>
<beam.beam.version>0.2.0-incubating</beam.beam.version>
<beam.beam.version>2.0.0</beam.beam.version>
<!-- library versions -->
<netty.version>4.1.1.Final</netty.version>
<servlet.api.version>3.1.0</servlet.api.version>
<commons.exec.version>1.3</commons.exec.version>
<interpreter.name>beam</interpreter.name>
</properties>
<dependencies>
@ -211,6 +212,14 @@
<version>${beam.beam.version}</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-flink_${scala.binary.version}</artifactId>
<version>${beam.beam.version}</version>
<exclusions>
</exclusions>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
@ -232,69 +241,25 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

16
beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java
View file

@ -17,12 +17,6 @@
package org.apache.zeppelin.beam;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -30,9 +24,14 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
/**
* Beam interpreter
*
*/
public class BeamInterpreter extends Interpreter {
@ -92,7 +91,8 @@ public class BeamInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return Collections.emptyList();
}

26
beam/src/main/java/org/apache/zeppelin/beam/StaticRepl.java
View file

@ -17,6 +17,12 @@
package org.apache.zeppelin.beam;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.model.JavaClass;
import com.thoughtworks.qdox.model.JavaSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.tools.Diagnostic;
import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
@ -24,14 +30,6 @@ import javax.tools.JavaCompiler.CompilationTask;
import javax.tools.JavaFileObject;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.model.JavaClass;
import com.thoughtworks.qdox.model.JavaSource;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
@ -44,9 +42,7 @@ import java.util.Arrays;
import java.util.List;
/**
*
* StaticRepl for compling the java code in memory
*
*/
public class StaticRepl {
static Logger logger = LoggerFactory.getLogger(StaticRepl.class);
@ -70,7 +66,7 @@ public class StaticRepl {
for (int j = 0; j < classes.get(i).getMethods().size(); j++) {
if (classes.get(i).getMethods().get(j).getName().equals("main") && classes.get(i)
.getMethods().get(j).isStatic()) {
.getMethods().get(j).isStatic()) {
mainClassName = classes.get(i).getName();
hasMain = true;
break;
@ -133,12 +129,12 @@ public class StaticRepl {
try {
// creating new class loader
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] { new File("").toURI()
.toURL() });
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[]{new File("").toURI()
.toURL()});
// execute the Main method
Class.forName(generatedClassName, true, classLoader)
.getDeclaredMethod("main", new Class[] { String[].class })
.invoke(null, new Object[] { null });
.getDeclaredMethod("main", new Class[]{String[].class})
.invoke(null, new Object[]{null});
System.out.flush();
System.err.flush();

6
beam/src/main/resources/interpreter-setting.json
View file

@ -19,13 +19,15 @@
"envName": "ZEPPELIN_SCIO_ARGZ",
"propertyName": "zeppelin.scio.argz",
"defaultValue": "--runner=InProcessPipelineRunner",
"description": "Scio interpreter wide arguments"
"description": "Scio interpreter wide arguments",
"type": "textarea"
},
"zeppelin.scio.maxResult": {
"envName": "ZEPPELIN_SCIO_MAXRESULT",
"propertyName": "zeppelin.scio.maxResult",
"defaultValue": "1000",
"description": "Max number of SCollection results to display."
"description": "Max number of SCollection results to display.",
"type": "number"
}
},
"editor": {

16
beam/src/main/test/org/apache/zeppelin/beam/BeamInterpreterTest.java → beam/src/test/org/apache/zeppelin/beam/BeamInterpreterTest.java
View file

@ -17,21 +17,20 @@
package org.apache.zeppelin.beam;
import static org.junit.Assert.assertEquals;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
/**
*
* BeamInterpreterTest
*
*/
public class BeamInterpreterTest {
@ -43,8 +42,7 @@ public class BeamInterpreterTest {
Properties p = new Properties();
beam = new BeamInterpreter(p);
beam.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null,
null);
context = InterpreterContext.builder().build();
}
@AfterClass

85
bigquery/README.md
View file

@ -1,10 +1,6 @@
# Overview
BigQuery interpreter for Apache Zeppelin
# Pre requisities
You can follow the instructions at [Apache Zeppelin on Dataproc](https://github.com/GoogleCloudPlatform/dataproc-initialization-actions/blob/master/apache-zeppelin/README.MD) to bring up Zeppelin on Google dataproc.
You could also install and bring up Zeppelin on Google compute Engine.
# Unit Tests
BigQuery Unit tests are excluded as these tests depend on the BigQuery external service. This is because BigQuery does not have a local mock at this point.
@ -14,34 +10,6 @@ If you like to run these tests manually, please follow the following steps:
* Copy the project ID that you created and add it to the property "projectId" in `resources/constants.json`
* Run the command mvn <options> -Dbigquery.text.exclude='' test -pl bigquery -am
# Interpreter Configuration
Configure the following properties during Interpreter creation.
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Default Value</th>
<th>Description</th>
</tr>
<tr>
<td>zeppelin.bigquery.project_id</td>
<td> </td>
<td>Google Project Id</td>
</tr>
<tr>
<td>zeppelin.bigquery.wait_time</td>
<td>5000</td>
<td>Query Timeout in Milliseconds</td>
</tr>
<tr>
<td>zeppelin.bigquery.max_no_of_rows</td>
<td>100000</td>
<td>Max result set size</td>
</tr>
</table>
# Connection
The Interpreter opens a connection with the BigQuery Service using the supplied Google project ID and the compute environment variables.
@ -51,59 +19,6 @@ The Interpreter opens a connection with the BigQuery Service using the supplied
We have used the curated veneer version of the Java APIs versus [Idiomatic Java client] (https://github.com/GoogleCloudPlatform/gcloud-java/tree/master/gcloud-java-bigquery) to build the interpreter. This is mainly for usability reasons.
# Enabling the BigQuery Interpreter
In a notebook, to enable the **BigQuery** interpreter, click the **Gear** icon and select **bigquery**.
# Using the BigQuery Interpreter
In a paragraph, use `%bigquery.sql` to select the **BigQuery** interpreter and then input SQL statements against your datasets stored in BigQuery.
You can use [BigQuery SQL Reference](https://cloud.google.com/bigquery/query-reference) to build your own SQL.
For Example, SQL to query for top 10 departure delays across airports using the flights public dataset
```bash
%bigquery.sql
SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
FROM [bigquery-samples:airline_ontime_data.flights]
group by departure_airport
order by 2 desc
limit 10
```
Another Example, SQL to query for most commonly used java packages from the github data hosted in BigQuery
```bash
%bigquery.sql
SELECT
package,
COUNT(*) count
FROM (
SELECT
REGEXP_EXTRACT(line, r' ([a-z0-9\._]*)\.') package,
id
FROM (
SELECT
SPLIT(content, '\n') line,
id
FROM
[bigquery-public-data:github_repos.sample_contents]
WHERE
content CONTAINS 'import'
AND sample_path LIKE '%.java'
HAVING
LEFT(line, 6)='import' )
GROUP BY
package,
id )
GROUP BY
1
ORDER BY
count DESC
LIMIT
40
```
# Sample Screenshot
![Zeppelin BigQuery](https://cloud.githubusercontent.com/assets/10060731/16938817/b9213ea0-4db6-11e6-8c3b-8149a0bdf874.png)

94
bigquery/pom.xml
View file

@ -21,15 +21,16 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-bigquery</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: BigQuery interpreter</name>
<properties>
@ -41,6 +42,7 @@
<!-- library versions -->
<bigquery.api.version>v2-rev265-1.21.0</bigquery.api.version>
<gson.version>2.6</gson.version>
<interpreter.name>bigquery</interpreter.name>
</properties>
<dependencies>
@ -99,12 +101,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
@ -116,63 +118,29 @@
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>
org.apache.zeppelin.bigquery.BigQueryInterpreter
</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>
org.apache.zeppelin.bigquery.BigQueryInterpreter
</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

124
bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
View file

@ -16,62 +16,45 @@
package org.apache.zeppelin.bigquery;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.BigqueryScopes;
import com.google.api.client.json.GenericJson;
import com.google.api.services.bigquery.Bigquery.Datasets;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.util.Joiner;
import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults;
import com.google.api.services.bigquery.BigqueryRequest;
import com.google.api.services.bigquery.model.DatasetList;
import com.google.api.services.bigquery.BigqueryScopes;
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobCancelResponse;
import com.google.api.services.bigquery.model.QueryRequest;
import com.google.api.services.bigquery.model.QueryResponse;
import com.google.api.services.bigquery.model.TableCell;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults;
import com.google.api.services.bigquery.model.GetQueryResultsResponse;
import com.google.api.services.bigquery.model.QueryRequest;
import com.google.api.services.bigquery.model.QueryResponse;
import com.google.api.services.bigquery.model.JobCancelResponse;
import com.google.gson.Gson;
import com.google.common.base.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import java.io.PrintStream;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
/**
* BigQuery interpreter for Zeppelin.
@ -95,11 +78,8 @@ import java.util.NoSuchElementException;
* </p>
*
*/
public class BigQueryInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(BigQueryInterpreter.class);
private static Logger logger = LoggerFactory.getLogger(BigQueryInterpreter.class);
private static final char NEWLINE = '\n';
private static final char TAB = '\t';
private static Bigquery service = null;
@ -109,6 +89,7 @@ public class BigQueryInterpreter extends Interpreter {
static final String PROJECT_ID = "zeppelin.bigquery.project_id";
static final String WAIT_TIME = "zeppelin.bigquery.wait_time";
static final String MAX_ROWS = "zeppelin.bigquery.max_no_of_rows";
static final String SQL_DIALECT = "zeppelin.bigquery.sql_dialect";
private static String jobId = null;
private static String projectId = null;
@ -127,7 +108,6 @@ public class BigQueryInterpreter extends Interpreter {
super(property);
}
//Function to return valid BigQuery Service
@Override
public void open() {
@ -165,23 +145,24 @@ public class BigQueryInterpreter extends Interpreter {
//Function that generates and returns the schema and the rows as string
public static String printRows(final GetQueryResultsResponse response) {
StringBuilder msg = null;
msg = new StringBuilder();
StringBuilder msg = new StringBuilder();
try {
List<String> schemNames = new ArrayList<String>();
for (TableFieldSchema schem: response.getSchema().getFields()) {
msg.append(schem.getName());
msg.append(TAB);
}
schemNames.add(schem.getName());
}
msg.append(Joiner.on(TAB).join(schemNames));
msg.append(NEWLINE);
for (TableRow row : response.getRows()) {
List<String> fieldValues = new ArrayList<String>();
for (TableCell field : row.getF()) {
msg.append(field.getV().toString());
msg.append(TAB);
fieldValues.add(field.getV().toString());
}
msg.append(Joiner.on(TAB).join(fieldValues));
msg.append(NEWLINE);
}
return msg.toString();
} catch ( NullPointerException ex ) {
} catch (NullPointerException ex) {
throw new NullPointerException("SQL Execution returned an error!");
}
}
@ -206,7 +187,7 @@ public class BigQueryInterpreter extends Interpreter {
class PageIterator implements Iterator<T> {
private BigqueryRequest<T> request;
private boolean hasNext = true;
public PageIterator(final BigqueryRequest<T> requestTemplate) {
PageIterator(final BigqueryRequest<T> requestTemplate) {
this.request = requestTemplate;
}
public boolean hasNext() {
@ -245,10 +226,23 @@ public class BigQueryInterpreter extends Interpreter {
String projId = getProperty(PROJECT_ID);
long wTime = Long.parseLong(getProperty(WAIT_TIME));
long maxRows = Long.parseLong(getProperty(MAX_ROWS));
String sqlDialect = getProperty(SQL_DIALECT, "").toLowerCase();
Boolean useLegacySql;
switch (sqlDialect) {
case "standardsql":
useLegacySql = false;
break;
case "legacysql":
useLegacySql = true;
break;
default:
// Enable query prefix like '#standardSQL' if specified
useLegacySql = null;
}
Iterator<GetQueryResultsResponse> pages;
try {
pages = run(sql, projId, wTime, maxRows);
} catch ( IOException ex ) {
pages = run(sql, projId, wTime, maxRows, useLegacySql);
} catch (IOException ex) {
logger.error(ex.getMessage());
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
@ -257,20 +251,25 @@ public class BigQueryInterpreter extends Interpreter {
finalmessage.append(printRows(pages.next()));
}
return new InterpreterResult(Code.SUCCESS, finalmessage.toString());
} catch ( NullPointerException ex ) {
} catch (NullPointerException ex) {
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
//Function to run the SQL on bigQuery service
public static Iterator<GetQueryResultsResponse> run(final String queryString,
final String projId, final long wTime, final long maxRows)
throws IOException {
public static Iterator<GetQueryResultsResponse> run(final String queryString,
final String projId, final long wTime, final long maxRows, Boolean useLegacySql)
throws IOException {
try {
QueryResponse query = service.jobs().query(
projId,
new QueryRequest().setTimeoutMs(wTime).setQuery(queryString).setMaxResults(maxRows))
.execute();
logger.info("Use legacy sql: {}", useLegacySql);
QueryResponse query;
query = service
.jobs()
.query(
projId,
new QueryRequest().setTimeoutMs(wTime)
.setUseLegacySql(useLegacySql).setQuery(queryString)
.setMaxResults(maxRows)).execute();
jobId = query.getJobReference().getJobId();
projectId = query.getJobReference().getProjectId();
GetQueryResults getRequest = service.jobs().getQueryResults(
@ -284,7 +283,6 @@ public class BigQueryInterpreter extends Interpreter {
@Override
public void close() {
logger.info("Close bqsql connection!");
service = null;
@ -314,7 +312,6 @@ public class BigQueryInterpreter extends Interpreter {
@Override
public void cancel(InterpreterContext context) {
logger.info("Trying to Cancel current query statement.");
if (service != null && jobId != null && projectId != null) {
@ -332,7 +329,8 @@ public class BigQueryInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return NO_COMPLETION;
}
}

19
bigquery/src/main/resources/interpreter-setting.json
View file

@ -8,24 +8,35 @@
"envName": null,
"propertyName": "zeppelin.bigquery.project_id",
"defaultValue": " ",
"description": "Google Project ID"
"description": "Google Project ID",
"type": "string"
},
"zeppelin.bigquery.wait_time": {
"envName": null,
"propertyName": "zeppelin.bigquery.wait_time",
"defaultValue": "5000",
"description": "Query timeout in Milliseconds"
"description": "Query timeout in Milliseconds",
"type": "number"
},
"zeppelin.bigquery.max_no_of_rows": {
"envName": null,
"propertyName": "zeppelin.bigquery.max_no_of_rows",
"defaultValue": "100000",
"description": "Maximum number of rows to fetch from BigQuery"
"description": "Maximum number of rows to fetch from BigQuery",
"type": "number"
},
"zeppelin.bigquery.sql_dialect": {
"envName": null,
"propertyName": "zeppelin.bigquery.sql_dialect",
"defaultValue": "",
"description": "BigQuery SQL dialect (standardSQL or legacySQL). If empty, query prefix like '#standardSQL' can be used.",
"type": "string"
}
},
"editor": {
"language": "sql",
"editOnDblClick": false
"editOnDblClick": false,
"completionSupport": false
}
}
]

64
bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java
View file

@ -14,41 +14,27 @@
* limitations under the License.
*/
package org.apache.zeppelin.bigquery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
import com.google.gson.Gson;
import com.google.gson.JsonIOException;
import com.google.gson.JsonSyntaxException;
import org.junit.Before;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
public class BigQueryInterpreterTest {
protected static class Constants {
private String projectId;
private String oneQuery;
@ -65,17 +51,15 @@ public class BigQueryInterpreterTest {
public String getWrong() {
return wrongQuery;
}
}
@SuppressWarnings("checkstyle:abbreviationaswordinname")
protected static Constants CONSTANTS = null;
protected static Constants constants = null;
public BigQueryInterpreterTest()
throws JsonSyntaxException, JsonIOException, FileNotFoundException {
if (CONSTANTS == null) {
if (constants == null) {
InputStream is = this.getClass().getResourceAsStream("/constants.json");
CONSTANTS = (new Gson()).<Constants>fromJson(new InputStreamReader(is), Constants.class);
constants = (new Gson()).<Constants>fromJson(new InputStreamReader(is), Constants.class);
}
}
@ -87,32 +71,44 @@ public class BigQueryInterpreterTest {
@Before
public void setUp() throws Exception {
Properties p = new Properties();
p.setProperty("zeppelin.bigquery.project_id", CONSTANTS.getProjectId());
p.setProperty("zeppelin.bigquery.project_id", constants.getProjectId());
p.setProperty("zeppelin.bigquery.wait_time", "5000");
p.setProperty("zeppelin.bigquery.max_no_of_rows", "100");
p.setProperty("zeppelin.bigquery.sql_dialect", "");
intpGroup = new InterpreterGroup();
bqInterpreter = new BigQueryInterpreter(p);
bqInterpreter.setInterpreterGroup(intpGroup);
bqInterpreter.open();
}
@Test
public void sqlSuccess() {
InterpreterResult ret = bqInterpreter.interpret(CONSTANTS.getOne(), context);
InterpreterResult ret = bqInterpreter.interpret(constants.getOne(), context);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(ret.message().get(0).getType(), InterpreterResult.Type.TABLE);
}
@Test
public void badSqlSyntaxFails() {
InterpreterResult ret = bqInterpreter.interpret(CONSTANTS.getWrong(), context);
InterpreterResult ret = bqInterpreter.interpret(constants.getWrong(), context);
assertEquals(InterpreterResult.Code.ERROR, ret.code());
}
@Test
public void testWithQueryPrefix() {
InterpreterResult ret = bqInterpreter.interpret(
"#standardSQL\n WITH t AS (select 1) SELECT * FROM t", context);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
}
@Test
public void testInterpreterOutputData() {
InterpreterResult ret = bqInterpreter.interpret("SELECT 1 AS col1, 2 AS col2", context);
String[] lines = ret.message().get(0).getData().split("\\n");
assertEquals(2, lines.length);
assertEquals("col1\tcol2", lines[0]);
assertEquals("1\t2", lines[1]);
}
}

8
bin/common.cmd
View file

@ -71,6 +71,14 @@ if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)
if defined ZEPPELIN_JMX_ENABLE (
if not defined ZEPPELIN_JMX_PORT (
set ZEPPELIN_JMX_PORT="9996"
}
set JMX_JAVA_OPTS=" -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
set ZEPPELIN_JAVA_OPTS=%JMX_JAVA_OPTS% %ZEPPELIN_JAVA_OPTS
)
if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (

18
bin/common.sh
View file

@ -121,8 +121,24 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
if [[ x"${ZEPPELIN_JMX_ENABLE}" == x"true" ]]; then
if [[ -z "${ZEPPELIN_JMX_PORT}" ]]; then
ZEPPELIN_JMX_PORT="9996"
fi
JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote"
JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.port=${ZEPPELIN_JMX_PORT}"
JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.authenticate=false"
JMX_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.ssl=false"
JAVA_OPTS="${JMX_JAVA_OPTS} ${JAVA_OPTS}"
fi
export JAVA_OPTS
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
else
JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties"
fi
export JAVA_INTP_OPTS

5
bin/interpreter.cmd
View file

@ -27,6 +27,7 @@ if /I "%~1"=="-d" (
set INTERPRETER_ID=%~n2
)
if /I "%~1"=="-p" set PORT=%~2
if /I "%~1"=="-c" set CALLBACK_HOST=%~2
if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2
shift
goto loop
@ -127,11 +128,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
if defined SPARK_SUBMIT (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%'
"%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" %PORT%
"%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT%
) else (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"
"%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% %PORT%
"%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT%
)
exit /b

75
bin/interpreter.sh
View file

@ -16,14 +16,15 @@
# limitations under the License.
#
bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
function usage() {
echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
}
while getopts "hp:d:l:v:u:" o; do
while getopts "hc:p:r:i:d:l:v:u:g:" o; do
case ${o} in
h)
usage
@ -32,8 +33,17 @@ while getopts "hp:d:l:v:u:" o; do
d)
INTERPRETER_DIR=${OPTARG}
;;
c)
CALLBACK_HOST=${OPTARG} # This will be used callback host
;;
p)
PORT=${OPTARG}
PORT=${OPTARG} # This will be used for callback port
;;
r)
INTP_PORT=${OPTARG} # This will be used for interpreter process port
;;
i)
INTP_GROUP_ID=${OPTARG} # This will be used for interpreter group id
;;
l)
LOCAL_INTERPRETER_REPO=${OPTARG}
@ -44,11 +54,9 @@ while getopts "hp:d:l:v:u:" o; do
;;
u)
ZEPPELIN_IMPERSONATE_USER="${OPTARG}"
if [[ -z "$ZEPPELIN_IMPERSONATE_CMD" ]]; then
ZEPPELIN_IMPERSONATE_RUN_CMD=`echo "ssh ${ZEPPELIN_IMPERSONATE_USER}@localhost" `
else
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
;;
g)
INTERPRETER_SETTING_NAME=${OPTARG}
;;
esac
done
@ -61,7 +69,7 @@ fi
. "${bin}/common.sh"
ZEPPELIN_INTP_CLASSPATH=""
ZEPPELIN_INTP_CLASSPATH="${CLASSPATH}"
# construct classpath
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
@ -85,11 +93,21 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-${INTERPRETER_SETTING_NAME}-"
if [[ -z "$ZEPPELIN_IMPERSONATE_CMD" ]]; then
if [[ "${INTERPRETER_ID}" != "spark" || "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" == "false" ]]; then
ZEPPELIN_IMPERSONATE_RUN_CMD=`echo "ssh ${ZEPPELIN_IMPERSONATE_USER}@localhost" `
fi
else
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
fi
ZEPPELIN_LOGFILE+="${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log"
ZEPPELIN_LOGFILE+="${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log"
JAVA_INTP_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
@ -99,9 +117,14 @@ fi
# set spark related env variables
if [[ "${INTERPRETER_ID}" == "spark" ]]; then
# run kinit
if [[ -n "${ZEPPELIN_SERVER_KERBEROS_KEYTAB}" ]] && [[ -n "${ZEPPELIN_SERVER_KERBEROS_PRINCIPAL}" ]]; then
kinit -kt ${ZEPPELIN_SERVER_KERBEROS_KEYTAB} ${ZEPPELIN_SERVER_KERBEROS_PRINCIPAL}
fi
if [[ -n "${SPARK_HOME}" ]]; then
export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit"
SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)"
SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/spark-interpreter*.jar)"
# This will evantually passes SPARK_APP_JAR to classpath of SparkIMain
ZEPPELIN_INTP_CLASSPATH+=":${SPARK_APP_JAR}"
@ -134,7 +157,13 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
fi
unset PYSPARKPATH
export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
else
# autodetect HADOOP_CONF_HOME by heuristic
if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then
if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then
@ -143,13 +172,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export HADOOP_CONF_DIR="/etc/hadoop/conf"
fi
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
fi
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
if [[ -n "${HBASE_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}"
@ -184,9 +208,8 @@ fi
addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}"
CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
if [[ "${INTERPRETER_ID}" != "spark" || "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" == "false" ]]; then
suid="$(id -u ${ZEPPELIN_IMPERSONATE_USER})"
if [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
INTERPRETER_RUN_COMMAND=${ZEPPELIN_IMPERSONATE_RUN_CMD}" '"
@ -194,25 +217,23 @@ if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
INTERPRETER_RUN_COMMAND+=" source "${ZEPPELIN_CONF_DIR}'/zeppelin-env.sh;'
fi
fi
fi
fi
if [[ -n "${SPARK_SUBMIT}" ]]; then
if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}`
else
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}`
fi
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
else
INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} `
INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${CALLBACK_HOST} ${PORT} ${INTP_GROUP_ID} ${INTP_PORT}`
fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
INTERPRETER_RUN_COMMAND+="'"
fi
eval $INTERPRETER_RUN_COMMAND &
pid=$!
if [[ -z "${pid}" ]]; then
exit 1;
else

47
bin/stop-interpreter.sh Executable file
View file

@ -0,0 +1,47 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Stop Zeppelin Interpreter Processes
#
bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
. "${bin}/common.sh"
export ZEPPELIN_FORCE_STOP=1
ZEPPELIN_STOP_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.recovery.StopInterpreter
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/stop-interpreter.log"
JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
fi
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
addJarInDir "${ZEPPELIN_HOME}/lib"
addJarInDir "${ZEPPELIN_HOME}/lib/interpreter"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_STOP_INTERPRETER_MAIN ${@}

16
bin/zeppelin-daemon.sh
View file

@ -67,6 +67,10 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
# Add jdbc connector jar
# ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar"
@ -213,18 +217,6 @@ function stop() {
action_msg "${ZEPPELIN_NAME} stop" "${SET_OK}"
fi
fi
# list all pid that used in remote interpreter and kill them
for f in ${ZEPPELIN_PID_DIR}/*.pid; do
if [[ ! -f ${f} ]]; then
continue;
fi
pid=$(cat ${f})
wait_for_zeppelin_to_die $pid 20
$(rm -f ${f})
done
}
function find_zeppelin_process() {

8
bin/zeppelin.sh
View file

@ -71,7 +71,11 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}"
@ -83,4 +87,4 @@ if [[ ! -d "${ZEPPELIN_PID_DIR}" ]]; then
$(mkdir -p "${ZEPPELIN_PID_DIR}")
fi
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_SERVER "$@"
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:${ZEPPELIN_CLASSPATH} $ZEPPELIN_SERVER "$@"

65
cassandra/pom.xml
View file

@ -20,16 +20,16 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-cassandra_2.10</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Apache Cassandra interpreter</name>
<description>Zeppelin cassandra support</description>
@ -49,6 +49,7 @@
<plugin.scala.version>2.15.2</plugin.scala.version>
<plugin.scalatest.version>1.0</plugin.scalatest.version>
<plugin.scalate.version>1.7.1</plugin.scalate.version>
<interpreter.name>cassandra</interpreter.name>
</properties>
<dependencies>
@ -241,55 +242,21 @@
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

145
cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java
View file

@ -16,35 +16,43 @@
*/
package org.apache.zeppelin.cassandra;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.ProtocolOptions.Compression;
import com.datastax.driver.core.Session;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import static java.lang.Integer.parseInt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.KeyStore;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import static com.datastax.driver.core.ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS;
import static java.lang.Integer.parseInt;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.JdkSSLOptions;
import com.datastax.driver.core.ProtocolOptions.Compression;
import com.datastax.driver.core.Session;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
/**
* Interpreter for Apache Cassandra CQL query language
* Interpreter for Apache Cassandra CQL query language.
*/
public class CassandraInterpreter extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(CassandraInterpreter.class);
public static final String CASSANDRA_INTERPRETER_PARALLELISM = "cassandra.interpreter" +
".parallelism";
public static final String CASSANDRA_INTERPRETER_PARALLELISM =
"cassandra.interpreter.parallelism";
public static final String CASSANDRA_HOSTS = "cassandra.hosts";
public static final String CASSANDRA_PORT = "cassandra.native.port";
public static final String CASSANDRA_PROTOCOL_VERSION = "cassandra.protocol.version";
@ -59,21 +67,21 @@ public class CassandraInterpreter extends Interpreter {
public static final String CASSANDRA_SPECULATIVE_EXECUTION_POLICY =
"cassandra.speculative.execution.policy";
public static final String CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS =
"cassandra.max.schema.agreement.wait.second";
"cassandra.max.schema.agreement.wait.second";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL =
"cassandra.pooling.new.connection.threshold.local";
"cassandra.pooling.new.connection.threshold.local";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE =
"cassandra.pooling.new.connection.threshold.remote";
"cassandra.pooling.new.connection.threshold.remote";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.max.connection.per.host.local";
"cassandra.pooling.max.connection.per.host.local";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.max.connection.per.host.remote";
"cassandra.pooling.max.connection.per.host.remote";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.core.connection.per.host.local";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.core.connection.per.host.remote";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL =
"cassandra.pooling.max.request.per.connection.local";
"cassandra.pooling.max.request.per.connection.local";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE =
"cassandra.pooling.max.request.per.connection.remote";
public static final String CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS =
@ -106,6 +114,13 @@ public class CassandraInterpreter extends Interpreter {
"cassandra.socket.soLinger";
public static final String CASSANDRA_SOCKET_TCP_NO_DELAY =
"cassandra.socket.tcp.no_delay";
public static final String CASSANDRA_WITH_SSL =
"cassandra.ssl.enabled";
public static final String CASSANDRA_TRUSTSTORE_PATH =
"cassandra.ssl.truststore.path";
public static final String CASSANDRA_TRUSTSTORE_PASSWORD =
"cassandra.ssl.truststore.password";
public static final String DEFAULT_HOST = "localhost";
public static final String DEFAULT_PORT = "9042";
@ -116,14 +131,14 @@ public class CassandraInterpreter extends Interpreter {
public static final String DEFAULT_CREDENTIAL = "none";
public static final String DEFAULT_POLICY = "DEFAULT";
public static final String DEFAULT_PARALLELISM = "10";
static String DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100";
static String DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "100";
static String DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2";
static String DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1";
static String DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8";
static String DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2";
static String DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024";
static String DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256";
static String defaultNewConnectionThresholdLocal = "100";
static String defaultNewConnectionThresholdRemote = "100";
static String defaultCoreConnectionPerHostLocal = "2";
static String defaultCoreConnectionPerHostRemote = "1";
static String defaultMaxConnectionPerHostLocal = "8";
static String defaultMaxConnectionPerHostRemote = "2";
static String defaultMaxRequestPerConnectionLocal = "1024";
static String defaultMaxRequestPerConnectionRemote = "256";
public static final String DEFAULT_IDLE_TIMEOUT = "120";
public static final String DEFAULT_POOL_TIMEOUT = "5000";
public static final String DEFAULT_HEARTBEAT_INTERVAL = "30";
@ -143,6 +158,7 @@ public class CassandraInterpreter extends Interpreter {
public static final List NO_COMPLETION = new ArrayList<>();
InterpreterLogic helper;
Cluster.Builder clusterBuilder;
Cluster cluster;
Session session;
private JavaDriverConfig driverConfig = new JavaDriverConfig();
@ -162,29 +178,58 @@ public class CassandraInterpreter extends Interpreter {
}
LOGGER.info("Bootstrapping Cassandra Java Driver to connect to " + hosts.toString() +
"on port " + port);
"on port " + port);
Compression compression = driverConfig.getCompressionProtocol(this);
cluster = Cluster.builder()
.addContactPoints(addresses)
.withPort(port)
.withProtocolVersion(driverConfig.getProtocolVersion(this))
.withClusterName(getProperty(CASSANDRA_CLUSTER_NAME))
.withCompression(compression)
.withCredentials(getProperty(CASSANDRA_CREDENTIALS_USERNAME),
getProperty(CASSANDRA_CREDENTIALS_PASSWORD))
.withLoadBalancingPolicy(driverConfig.getLoadBalancingPolicy(this))
.withRetryPolicy(driverConfig.getRetryPolicy(this))
.withReconnectionPolicy(driverConfig.getReconnectionPolicy(this))
.withSpeculativeExecutionPolicy(driverConfig.getSpeculativeExecutionPolicy(this))
.withMaxSchemaAgreementWaitSeconds(
parseInt(getProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS)))
.withPoolingOptions(driverConfig.getPoolingOptions(this))
.withQueryOptions(driverConfig.getQueryOptions(this))
.withSocketOptions(driverConfig.getSocketOptions(this))
.build();
clusterBuilder = Cluster.builder()
.addContactPoints(addresses)
.withPort(port)
.withProtocolVersion(driverConfig.getProtocolVersion(this))
.withClusterName(getProperty(CASSANDRA_CLUSTER_NAME))
.withCompression(compression)
.withCredentials(getProperty(CASSANDRA_CREDENTIALS_USERNAME),
getProperty(CASSANDRA_CREDENTIALS_PASSWORD))
.withLoadBalancingPolicy(driverConfig.getLoadBalancingPolicy(this))
.withRetryPolicy(driverConfig.getRetryPolicy(this))
.withReconnectionPolicy(driverConfig.getReconnectionPolicy(this))
.withSpeculativeExecutionPolicy(driverConfig.getSpeculativeExecutionPolicy(this))
.withMaxSchemaAgreementWaitSeconds(
parseInt(getProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS)))
.withPoolingOptions(driverConfig.getPoolingOptions(this))
.withQueryOptions(driverConfig.getQueryOptions(this))
.withSocketOptions(driverConfig.getSocketOptions(this));
final String runWithSSL = getProperty(CASSANDRA_WITH_SSL);
if (runWithSSL != null && runWithSSL.equals("true")) {
LOGGER.debug("Cassandra Interpreter: Using SSL");
try {
final SSLContext sslContext;
{
final KeyStore trustStore = KeyStore.getInstance("JKS");
final InputStream stream = Files.newInputStream(Paths.get(
getProperty(CASSANDRA_TRUSTSTORE_PATH)));
trustStore.load(stream, getProperty(CASSANDRA_TRUSTSTORE_PASSWORD).toCharArray());
final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(
TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(trustStore);
sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagerFactory.getTrustManagers(), null);
}
clusterBuilder = clusterBuilder.withSSL(JdkSSLOptions.builder()
.withSSLContext(sslContext)
.build());
} catch (Exception e) {
LOGGER.error(e.toString());
}
} else {
LOGGER.debug("Cassandra Interpreter: Not using SSL");
}
cluster = clusterBuilder.build();
session = cluster.connect();
helper = new InterpreterLogic(session);
}
@ -202,7 +247,6 @@ public class CassandraInterpreter extends Interpreter {
@Override
public void cancel(InterpreterContext context) {
}
@Override
@ -216,7 +260,8 @@ public class CassandraInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return NO_COMPLETION;
}

2
cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java
View file

@ -17,7 +17,7 @@
package org.apache.zeppelin.cassandra;
/**
* Parsing Exception for Cassandra CQL statement
* Parsing Exception for Cassandra CQL statement.
*/
public class ParsingException extends RuntimeException{
public ParsingException(String message) {

116
cassandra/src/main/resources/interpreter-setting.json
View file

@ -8,187 +8,239 @@
"envName": null,
"propertyName": "cassandra.hosts",
"defaultValue": "localhost",
"description": "Comma separated Cassandra hosts (DNS name or IP address). Default = localhost. Ex: '192.168.0.12,node2,node3'"
"description": "Comma separated Cassandra hosts (DNS name or IP address). Default = localhost. Ex: '192.168.0.12,node2,node3'",
"type": "textarea"
},
"cassandra.native.port": {
"envName": null,
"propertyName": "cassandra.native.port",
"defaultValue": "9042",
"description": "Cassandra native port. Default = 9042"
"description": "Cassandra native port. Default = 9042",
"type": "number"
},
"cassandra.protocol.version": {
"envName": null,
"propertyName": "cassandra.protocol.version",
"defaultValue": "4",
"description": "Cassandra protocol version. Default = 4"
"description": "Cassandra protocol version. Default = 4",
"type": "string"
},
"cassandra.cluster": {
"envName": null,
"propertyName": "cassandra.cluster",
"defaultValue": "Test Cluster",
"description": "Cassandra cluster name. Default = 'Test Cluster'"
"description": "Cassandra cluster name. Default = 'Test Cluster'",
"type": "string"
},
"cassandra.keyspace": {
"envName": null,
"propertyName": "cassandra.keyspace",
"defaultValue": "system",
"description": "Cassandra keyspace name. Default = 'system'"
"description": "Cassandra keyspace name. Default = 'system'",
"type": "string"
},
"cassandra.compression.protocol": {
"envName": null,
"propertyName": "cassandra.compression.protocol",
"defaultValue": "NONE",
"description": "Cassandra compression protocol. Available values: NONE, SNAPPY, LZ4. Default = NONE"
"description": "Cassandra compression protocol. Available values: NONE, SNAPPY, LZ4. Default = NONE",
"type": "string"
},
"cassandra.credentials.username": {
"envName": null,
"propertyName": "cassandra.credentials.username",
"defaultValue": "none",
"description": "Cassandra credentials username. Default = 'none'"
"description": "Cassandra credentials username. Default = 'none'",
"type": "string"
},
"cassandra.credentials.password": {
"envName": null,
"propertyName": "cassandra.credentials.password",
"defaultValue": "none",
"description": "Cassandra credentials password. Default = 'none'"
"description": "Cassandra credentials password. Default = 'none'",
"type": "password"
},
"cassandra.load.balancing.policy": {
"envName": null,
"propertyName": "cassandra.load.balancing.policy",
"defaultValue": "DEFAULT",
"description": "Cassandra Load Balancing Policy. Default = new TokenAwarePolicy(new DCAwareRoundRobinPolicy())"
"description": "Cassandra Load Balancing Policy. Default = new TokenAwarePolicy(new DCAwareRoundRobinPolicy())",
"type": "string"
},
"cassandra.retry.policy": {
"envName": null,
"propertyName": "cassandra.retry.policy",
"defaultValue": "DEFAULT",
"description": "Cassandra Retry Policy. Default = DefaultRetryPolicy.INSTANCE"
"description": "Cassandra Retry Policy. Default = DefaultRetryPolicy.INSTANCE",
"type": "string"
},
"cassandra.reconnection.policy": {
"envName": null,
"propertyName": "cassandra.reconnection.policy",
"defaultValue": "DEFAULT",
"description": "Cassandra Reconnection Policy. Default = new ExponentialReconnectionPolicy(1000, 10 * 60 * 1000)"
"description": "Cassandra Reconnection Policy. Default = new ExponentialReconnectionPolicy(1000, 10 * 60 * 1000)",
"type": "string"
},
"cassandra.speculative.execution.policy": {
"envName": null,
"propertyName": "cassandra.speculative.execution.policy",
"defaultValue": "DEFAULT",
"description": "Cassandra Speculative Execution Policy. Default = NoSpeculativeExecutionPolicy.INSTANCE"
"description": "Cassandra Speculative Execution Policy. Default = NoSpeculativeExecutionPolicy.INSTANCE",
"type": "string"
},
"cassandra.interpreter.parallelism": {
"envName": null,
"propertyName": "cassandra.interpreter.parallelism",
"defaultValue": "10",
"description": "Cassandra interpreter parallelism.Default = 10"
"description": "Cassandra interpreter parallelism.Default = 10",
"type": "number"
},
"cassandra.max.schema.agreement.wait.second": {
"envName": null,
"propertyName": "cassandra.max.schema.agreement.wait.second",
"defaultValue": "10",
"description": "Cassandra max schema agreement wait in second.Default = ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS"
"description": "Cassandra max schema agreement wait in second.Default = ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS",
"type": "number"
},
"cassandra.pooling.new.connection.threshold.local": {
"envName": null,
"propertyName": "cassandra.pooling.new.connection.threshold.local",
"defaultValue": "100",
"description": "Cassandra new connection threshold local. Protocol V2 and below default = 100 Protocol V3 and above default = 800"
"description": "Cassandra new connection threshold local. Protocol V2 and below default = 100 Protocol V3 and above default = 800",
"type": "number"
},
"cassandra.pooling.new.connection.threshold.remote": {
"envName": null,
"propertyName": "cassandra.pooling.new.connection.threshold.remote",
"defaultValue": "100",
"description": "Cassandra new connection threshold remove. Protocol V2 and below default = 100 Protocol V3 and above default = 200"
"description": "Cassandra new connection threshold remove. Protocol V2 and below default = 100 Protocol V3 and above default = 200",
"type": "number"
},
"cassandra.pooling.core.connection.per.host.local": {
"envName": null,
"propertyName": "cassandra.pooling.core.connection.per.host.local",
"defaultValue": "2",
"description": "Cassandra core connection per host local. Protocol V2 and below default = 2 Protocol V3 and above default = 1"
"description": "Cassandra core connection per host local. Protocol V2 and below default = 2 Protocol V3 and above default = 1",
"type": "number"
},
"cassandra.pooling.core.connection.per.host.remote": {
"envName": null,
"propertyName": "cassandra.pooling.core.connection.per.host.remote",
"defaultValue": "1",
"description": "Cassandra core connection per host remove. Protocol V2 and below default = 1 Protocol V3 and above default = 1"
"description": "Cassandra core connection per host remove. Protocol V2 and below default = 1 Protocol V3 and above default = 1",
"type": "number"
},
"cassandra.pooling.max.connection.per.host.local": {
"envName": null,
"propertyName": "cassandra.pooling.max.connection.per.host.local",
"defaultValue": "8",
"description": "Cassandra max connection per host local. Protocol V2 and below default = 8 Protocol V3 and above default = 1"
"description": "Cassandra max connection per host local. Protocol V2 and below default = 8 Protocol V3 and above default = 1",
"type": "number"
},
"cassandra.pooling.max.connection.per.host.remote": {
"envName": null,
"propertyName": "cassandra.pooling.max.connection.per.host.remote",
"defaultValue": "2",
"description": "Cassandra max connection per host remote. Protocol V2 and below default = 2 Protocol V3 and above default = 1"
"description": "Cassandra max connection per host remote. Protocol V2 and below default = 2 Protocol V3 and above default = 1",
"type": "number"
},
"cassandra.pooling.max.request.per.connection.local": {
"envName": null,
"propertyName": "cassandra.pooling.max.request.per.connection.local",
"defaultValue": "1024",
"description": "Cassandra max request per connection local. Protocol V2 and below default = 128 Protocol V3 and above default = 1024"
"description": "Cassandra max request per connection local. Protocol V2 and below default = 128 Protocol V3 and above default = 1024",
"type": "number"
},
"cassandra.pooling.max.request.per.connection.remote": {
"envName": null,
"propertyName": "cassandra.pooling.max.request.per.connection.remote",
"defaultValue": "256",
"description": "Cassandra max request per connection remote. Protocol V2 and below default = 128 Protocol V3 and above default = 256"
"description": "Cassandra max request per connection remote. Protocol V2 and below default = 128 Protocol V3 and above default = 256",
"type": "number"
},
"cassandra.pooling.idle.timeout.seconds": {
"envName": null,
"propertyName": "cassandra.pooling.idle.timeout.seconds",
"defaultValue": "120",
"description": "Cassandra idle time out in seconds. Default = 120"
"description": "Cassandra idle time out in seconds. Default = 120",
"type": "number"
},
"cassandra.pooling.pool.timeout.millisecs": {
"envName": null,
"propertyName": "cassandra.pooling.pool.timeout.millisecs",
"defaultValue": "5000",
"description": "Cassandra pool time out in millisecs. Default = 5000"
"description": "Cassandra pool time out in millisecs. Default = 5000",
"type": "number"
},
"cassandra.pooling.heartbeat.interval.seconds": {
"envName": null,
"propertyName": "cassandra.pooling.heartbeat.interval.seconds",
"defaultValue": "30",
"description": "Cassandra pool heartbeat interval in secs. Default = 30"
"description": "Cassandra pool heartbeat interval in secs. Default = 30",
"type": "number"
},
"cassandra.query.default.consistency": {
"envName": null,
"propertyName": "cassandra.query.default.consistency",
"defaultValue": "ONE",
"description": "Cassandra query default consistency level. Default = ONE"
"description": "Cassandra query default consistency level. Default = ONE",
"type": "string"
},
"cassandra.query.default.serial.consistency": {
"envName": null,
"propertyName": "cassandra.query.default.serial.consistency",
"defaultValue": "SERIAL",
"description": "Cassandra query default serial consistency level. Default = SERIAL"
"description": "Cassandra query default serial consistency level. Default = SERIAL",
"type": "string"
},
"cassandra.query.default.fetchSize": {
"envName": null,
"propertyName": "cassandra.query.default.fetchSize",
"defaultValue": "5000",
"description": "Cassandra query default fetch size. Default = 5000"
"description": "Cassandra query default fetch size. Default = 5000",
"type": "number"
},
"cassandra.socket.connection.timeout.millisecs": {
"envName": null,
"propertyName": "cassandra.socket.connection.timeout.millisecs",
"defaultValue": "5000",
"description": "Cassandra socket default connection timeout in millisecs. Default = 5000"
"description": "Cassandra socket default connection timeout in millisecs. Default = 5000",
"type": "number"
},
"cassandra.socket.read.timeout.millisecs": {
"envName": null,
"propertyName": "cassandra.socket.read.timeout.millisecs",
"defaultValue": "12000",
"description": "Cassandra socket read timeout in millisecs. Default = 12000"
"description": "Cassandra socket read timeout in millisecs. Default = 12000",
"type": "number"
},
"cassandra.socket.tcp.no_delay": {
"envName": null,
"propertyName": "cassandra.socket.tcp.no_delay",
"defaultValue": "true",
"description": "Cassandra socket TCP no delay. Default = true"
"defaultValue": true,
"description": "Cassandra socket TCP no delay. Default = true",
"type": "checkbox"
},
"cassandra.ssl.enabled": {
"envName": null,
"propertyName": "cassandra.ssl.enabled",
"defaultValue": false,
"description": "Cassandra SSL",
"type": "checkbox"
},
"cassandra.ssl.truststore.path": {
"envName": null,
"propertyName": "cassandra.ssl.truststore.path",
"defaultValue": "none",
"description": "Cassandra truststore path. Default = none",
"type": "string"
},
"cassandra.ssl.truststore.password": {
"envName": null,
"propertyName": "cassandra.ssl.truststore.password",
"defaultValue": "none",
"description": "Cassandra truststore password. Default = none",
"type": "password"
}
},
"editor": {

2
cassandra/src/main/scala/org/apache/zeppelin/cassandra/InterpreterLogic.scala
View file

@ -30,7 +30,7 @@ import com.datastax.driver.core.exceptions.DriverException
import com.datastax.driver.core.policies.{LoggingRetryPolicy, FallthroughRetryPolicy, DowngradingConsistencyRetryPolicy, Policies}
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
import org.apache.zeppelin.display.AngularObjectRegistry
import org.apache.zeppelin.display.Input.ParamOption
import org.apache.zeppelin.display.ui.OptionInput.ParamOption
import org.apache.zeppelin.interpreter.InterpreterResult.Code
import org.apache.zeppelin.interpreter.{InterpreterException, InterpreterResult, InterpreterContext}
import org.slf4j.LoggerFactory

80
cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala
View file

@ -178,54 +178,54 @@ class JavaDriverConfig {
protocolVersion match {
case "1" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128"
defaultMaxConnectionPerHostLocal = "8"
defaultMaxConnectionPerHostRemote = "2"
defaultCoreConnectionPerHostLocal = "2"
defaultCoreConnectionPerHostRemote = "1"
defaultNewConnectionThresholdLocal = "100"
defaultNewConnectionThresholdRemote = "1"
defaultMaxRequestPerConnectionLocal = "128"
defaultMaxRequestPerConnectionRemote = "128"
return ProtocolVersion.V1
case "2" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128"
defaultMaxConnectionPerHostLocal = "8"
defaultMaxConnectionPerHostRemote = "2"
defaultCoreConnectionPerHostLocal = "2"
defaultCoreConnectionPerHostRemote = "1"
defaultNewConnectionThresholdLocal = "100"
defaultNewConnectionThresholdRemote = "1"
defaultMaxRequestPerConnectionLocal = "128"
defaultMaxRequestPerConnectionRemote = "128"
return ProtocolVersion.V2
case "3" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"
defaultMaxConnectionPerHostLocal = "1"
defaultMaxConnectionPerHostRemote = "1"
defaultCoreConnectionPerHostLocal = "1"
defaultCoreConnectionPerHostRemote = "1"
defaultNewConnectionThresholdLocal = "800"
defaultNewConnectionThresholdRemote = "200"
defaultMaxRequestPerConnectionLocal = "1024"
defaultMaxRequestPerConnectionRemote = "256"
return ProtocolVersion.V3
case "4" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"
defaultMaxConnectionPerHostLocal = "1"
defaultMaxConnectionPerHostRemote = "1"
defaultCoreConnectionPerHostLocal = "1"
defaultCoreConnectionPerHostRemote = "1"
defaultNewConnectionThresholdLocal = "800"
defaultNewConnectionThresholdRemote = "200"
defaultMaxRequestPerConnectionLocal = "1024"
defaultMaxRequestPerConnectionRemote = "256"
return ProtocolVersion.V4
case _ =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"
defaultMaxConnectionPerHostLocal = "1"
defaultMaxConnectionPerHostRemote = "1"
defaultCoreConnectionPerHostLocal = "1"
defaultCoreConnectionPerHostRemote = "1"
defaultNewConnectionThresholdLocal = "800"
defaultNewConnectionThresholdRemote = "200"
defaultMaxRequestPerConnectionLocal = "1024"
defaultMaxRequestPerConnectionRemote = "256"
return ProtocolVersion.NEWEST_SUPPORTED
}
}

1409
cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java
View file

File diff suppressed because it is too large Load diff

518
cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java
View file

@ -16,27 +16,23 @@
*/
package org.apache.zeppelin.cassandra;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import static java.util.Arrays.asList;
import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED;
import static com.datastax.driver.core.ConsistencyLevel.ALL;
import static com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL;
import static com.datastax.driver.core.ConsistencyLevel.ONE;
import static com.datastax.driver.core.ConsistencyLevel.QUORUM;
import static com.datastax.driver.core.ConsistencyLevel.SERIAL;
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.*;
import static org.mockito.Mockito.*;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.Input.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@ -47,302 +43,334 @@ import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import scala.Option;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.*;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import scala.Option;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.AnyBlock;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.Consistency;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.DowngradingRetryPolicy$;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.LoggingDefaultRetryPolicy$;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.QueryParameters;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.RequestTimeOut;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.RetryPolicy;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.SerialConsistency;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.SimpleStm;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.Timestamp;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@RunWith(MockitoJUnitRunner.class)
public class InterpreterLogicTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext intrContext;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext intrContext;
@Mock
private Session session;
@Mock
private Session session;
final InterpreterLogic helper = new InterpreterLogic(session);
final InterpreterLogic helper = new InterpreterLogic(session);
@Captor
ArgumentCaptor<ParamOption[]> optionsCaptor;
@Captor
ArgumentCaptor<ParamOption[]> optionsCaptor;
@Test
public void should_parse_input_string_block() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
@Test
public void should_parse_input_string_block() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
//When
final List<AnyBlock> anyBlocks = this.<AnyBlock>toJavaList(helper.parseInput(input));
//When
final List<AnyBlock> anyBlocks = this.<AnyBlock>toJavaList(helper.parseInput(input));
//Then
assertThat(anyBlocks).hasSize(1);
assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class);
}
//Then
assertThat(anyBlocks).hasSize(1);
assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class);
}
@Test
public void should_exception_while_parsing_input() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10";
@Test
public void should_exception_while_parsing_input() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10";
//When
expectedException.expect(InterpreterException.class);
expectedException.expectMessage("Error parsing input:\n" +
"\t'SELECT * FROM users LIMIT 10'\n" +
"Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
//When
expectedException.expect(InterpreterException.class);
expectedException.expectMessage("Error parsing input:\n" +
"\t'SELECT * FROM users LIMIT 10'\n" +
"Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
helper.parseInput(input);
}
helper.parseInput(input);
}
@Test
public void should_extract_variable_and_default_value() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo");
when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'");
@Test
public void should_extract_variable_and_default_value() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo");
when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'");
//When
final String actual = helper.maybeExtractVariables(
"SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'");
}
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'");
}
@Test
public void should_extract_variable_and_choices() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture()))
.thenReturn("'Jack'");
@Test
public void should_extract_variable_and_choices() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture())).thenReturn("'Jack'");
//When
final String actual = helper.maybeExtractVariables(
"SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}",
intrContext);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}", intrContext);
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'");
final List<ParamOption> paramOptions = asList(optionsCaptor.getValue());
assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'");
assertThat(paramOptions.get(1).getValue()).isEqualTo("'Jack'");
assertThat(paramOptions.get(2).getValue()).isEqualTo("'Smith'");
}
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'");
final List<ParamOption> paramOptions = asList(optionsCaptor.getValue());
assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'");
assertThat(paramOptions.get(1).getValue()).isEqualTo("'Jack'");
assertThat(paramOptions.get(2).getValue()).isEqualTo("'Smith'");
}
@Test
public void should_extract_no_variable() throws Exception {
//Given
GUI gui = mock(GUI.class);
when(intrContext.getGui()).thenReturn(gui);
@Test
public void should_extract_no_variable() throws Exception {
//Given
GUI gui = mock(GUI.class);
when(intrContext.getGui()).thenReturn(gui);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext);
//Then
verifyZeroInteractions(gui);
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo");
}
//Then
verifyZeroInteractions(gui);
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo");
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
angularObjectRegistry.add("id", "from_angular_registry", "noteId", "paragraphId");
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getNoteId()).thenReturn("noteId");
when(intrContext.getParagraphId()).thenReturn("paragraphId");
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
angularObjectRegistry.add("id", "from_angular_registry", "noteId", "paragraphId");
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getNoteId()).thenReturn("noteId");
when(intrContext.getParagraphId()).thenReturn("paragraphId");
//When
final String actual = helper.maybeExtractVariables(
"SELECT * FROM zeppelin.demo WHERE id='{{id=John}}'", intrContext);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo WHERE id='{{id=John}}'", intrContext);
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='from_angular_registry'");
verify(intrContext, never()).getGui();
}
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='from_angular_registry'");
verify(intrContext, never()).getGui();
}
@Test
public void should_error_if_incorrect_variable_definition() throws Exception {
//Given
@Test
public void should_error_if_incorrect_variable_definition() throws Exception {
//Given
//When
expectedException.expect(ParsingException.class);
expectedException.expectMessage("Invalid bound variable definition for " +
"'{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} " +
"WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'");
//When
expectedException.expect(ParsingException.class);
expectedException.expectMessage("Invalid bound variable definition for '{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'");
//Then
helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}",
intrContext);
}
//Then
helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
}
@Test
public void should_extract_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Consistency(ALL),
new Consistency(ONE));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
@Test
public void should_extract_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Consistency(ALL), new Consistency(ONE));
//Then
assertThat(actual.consistency().get()).isEqualTo(ALL);
}
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
@Test
public void should_extract_serial_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new SerialConsistency(SERIAL),
new SerialConsistency(LOCAL_SERIAL));
//Then
assertThat(actual.consistency().get()).isEqualTo(ALL);
}
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL);
}
@Test
public void should_extract_serial_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new SerialConsistency(SERIAL), new SerialConsistency(LOCAL_SERIAL));
@Test
public void should_extract_timestamp_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Timestamp(123L),
new Timestamp(456L));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL);
}
//Then
assertThat(actual.timestamp().get()).isEqualTo(123L);
}
@Test
public void should_extract_timestamp_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Timestamp(123L), new Timestamp(456L));
@Test
public void should_extract_retry_policy_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(DowngradingRetryPolicy$.MODULE$,
LoggingDefaultRetryPolicy$.MODULE$);
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.timestamp().get()).isEqualTo(123L);
}
//Then
assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$);
}
@Test
public void should_extract_retry_policy_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(DowngradingRetryPolicy$.MODULE$, LoggingDefaultRetryPolicy$.MODULE$);
@Test
public void should_extract_request_timeout_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new RequestTimeOut(100));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$);
}
//Then
assertThat(actual.requestTimeOut().get()).isEqualTo(100);
}
@Test
public void should_extract_request_timeout_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new RequestTimeOut(100));
@Test
public void should_generate_simple_statement() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//When
final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options,
intrContext);
//Then
assertThat(actual.requestTimeOut().get()).isEqualTo(100);
}
//Then
assertThat(actual).isNotNull();
assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;");
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
@Test
public void should_generate_simple_statement() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
@Test
public void should_generate_batch_statement() throws Exception {
//Given
Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;");
Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);");
Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;");
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
//When
final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options, intrContext);
//When
BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options,
toScalaList(asList(st1, st2, st3)));
//Then
assertThat(actual).isNotNull();
assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;");
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
//Then
assertThat(actual).isNotNull();
final List<Statement> statements = new ArrayList<>(actual.getStatements());
assertThat(statements).hasSize(3);
assertThat(statements.get(0)).isSameAs(st1);
assertThat(statements.get(1)).isSameAs(st2);
assertThat(statements.get(2)).isSameAs(st3);
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
@Test
public void should_generate_batch_statement() throws Exception {
//Given
Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;");
Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);");
Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;");
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
@Test
public void should_parse_bound_values() throws Exception {
//Given
String bs = "'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'";
//When
BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options, toScalaList(asList(st1, st2, st3)));
//When
final List<String> actual = this.<String>toJavaList(helper.parseBoundValues("ps", bs));
//Then
assertThat(actual).isNotNull();
final List<Statement> statements = new ArrayList<>(actual.getStatements());
assertThat(statements).hasSize(3);
assertThat(statements.get(0)).isSameAs(st1);
assertThat(statements.get(1)).isSameAs(st2);
assertThat(statements.get(2)).isSameAs(st3);
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
//Then
assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'",
"null", "true", "2014-06-12 34:00:34");
}
@Test
public void should_parse_bound_values() throws Exception {
//Given
String bs="'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'";
@Test
public void should_parse_simple_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01";
//When
final List<String> actual = this.<String>toJavaList(helper.parseBoundValues("ps", bs));
//When
final Date actual = helper.parseDate(dateString);
//Then
assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'",
"null", "true", "2014-06-12 34:00:34");
}
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
@Test
public void should_parse_simple_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01";
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
}
//When
final Date actual = helper.parseDate(dateString);
@Test
public void should_parse_accurate_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01.123";
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
//When
final Date actual = helper.parseDate(dateString);
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
}
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
@Test
public void should_parse_accurate_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01.123";
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123);
}
//When
final Date actual = helper.parseDate(dateString);
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList();
}
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123);
}
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList();
}
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
return scala.collection.JavaConversions.seqAsJavaList(list);
}
}
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
return scala.collection.JavaConversions.seqAsJavaList(list);
}
}

33
conf/log4j.properties2 Normal file
View file

@ -0,0 +1,33 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger = INFO, dailyfile
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
log4j.appender.dailyfile.DatePattern=.yyyy-MM-dd
log4j.appender.dailyfile.DEBUG = INFO
log4j.appender.dailyfile = org.apache.log4j.DailyRollingFileAppender
log4j.appender.dailyfile.File = ${zeppelin.log.file}
log4j.appender.dailyfile.layout = org.apache.log4j.PatternLayout
log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
log4j.logger.org.apache.zeppelin.python=DEBUG
log4j.logger.org.apache.zeppelin.spark=DEBUG

23
conf/log4j_yarn_cluster.properties Normal file
View file

@ -0,0 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger = INFO, stdout
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n

52
conf/shiro.ini.template
View file

@ -18,7 +18,8 @@
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password1, admin
# To enable admin user, uncomment the following line and set an appropriate password.
#admin = password1, admin
user1 = password2, role1, role2
user2 = password3, role3
user3 = password4, role2
@ -43,7 +44,7 @@ user3 = password4, role2
#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
#ldapRealm.contextFactory.url = ldap://ldap.test.com:389
#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
#ldapRealm.contextFactory.authenticationMechanism = simple
### A sample PAM configuration
#pamRealm=org.apache.zeppelin.realm.PamRealm
@ -55,12 +56,34 @@ user3 = password4, role2
#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
#securityManager.realms = $zeppelinHubRealm
## A same for configuring Knox SSO Realm
#knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm
#knoxJwtRealm.providerUrl = https://domain.example.com/
#knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html
#knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout
#knoxJwtRealm.logoutAPI = true
#knoxJwtRealm.redirectParam = originalUrl
#knoxJwtRealm.cookieName = hadoop-jwt
#knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem
#
#knoxJwtRealm.groupPrincipalMapping = group.principal.mapping
#knoxJwtRealm.principalMapping = principal.mapping
#authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
### If caching of user is required then uncomment below lines
#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
#securityManager.cacheManager = $cacheManager
### Enables 'HttpOnly' flag in Zeppelin cookies
cookie = org.apache.shiro.web.servlet.SimpleCookie
cookie.name = JSESSIONID
cookie.httpOnly = true
### Uncomment the below line only when Zeppelin is running over HTTPS
#cookie.secure = true
sessionManager.sessionIdCookie = $cookie
securityManager.sessionManager = $sessionManager
# 86,400,000 milliseconds = 24 hour
securityManager.sessionManager.globalSessionTimeout = 86400000
@ -73,14 +96,25 @@ role3 = *
admin = *
[urls]
# This section is used for url-based security.
# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide.
# This section is used for url-based security. For details see the shiro.ini documentation.
#
# You can secure interpreter, configuration and credential information by urls.
# Comment or uncomment the below urls that you want to hide:
# anon means the access is anonymous.
# authc means Form based Auth Security
# To enfore security, comment the line below and uncomment the next one
# authc means form based auth Security.
#
# IMPORTANT: Order matters: URL path expressions are evaluated against an incoming request
# in the order they are defined and the FIRST MATCH WINS.
#
# To allow anonymous access to all but the stated urls,
# uncomment the line second last line (/** = anon) and comment the last line (/** = authc)
#
/api/version = anon
#/api/interpreter/** = authc, roles[admin]
#/api/configurations/** = authc, roles[admin]
#/api/credential/** = authc, roles[admin]
# Allow all authenticated users to restart interpreters on a notebook page.
# Comment out the following line if you would like to authorize only admin users to restart interpreters.
/api/interpreter/setting/restart/** = authc
/api/interpreter/** = authc, roles[admin]
/api/configurations/** = authc, roles[admin]
/api/credential/** = authc, roles[admin]
#/** = anon
/** = authc

6
conf/zeppelin-env.cmd.template
View file

@ -22,6 +22,8 @@ REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set Z
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default -Xmx1024m -Xms1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
REM set ZEPPELIN_JMX_ENABLE REM Enable JMX feature by defining it like "true"
REM set ZEPPELIN_JMX_PORT REM Port number which JMX uses. Default: "9996"
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.
@ -38,6 +40,10 @@ REM set ZEPPELIN_NOTEBOOK_S3_SSE REM Server-side encryption enable
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading
REM set ZEPPELIN_HELIUM_NODE_INSTALLER_URL REM Remote Node installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_NPM_INSTALLER_URL REM Remote Npm installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL REM Remote Yarn package installer url for Helium dependency loader
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?

21
conf/zeppelin-env.sh.template
View file

@ -23,6 +23,8 @@
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
# export ZEPPELIN_SSL_PORT # ssl port (used when ssl environment variable is set to true)
# export ZEPPELIN_JMX_ENABLE # Enable JMX feature by defining "true"
# export ZEPPELIN_JMX_PORT # Port number which JMX uses. Default: "9996"
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.
@ -30,21 +32,40 @@
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false"
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region
# export ZEPPELIN_NOTEBOOK_S3_SSE # Server-side encryption enabled for notebooks
# export ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR # GCS "directory" (prefix) under which notebooks are saved. E.g. gs://example-bucket/path/to/dir
# export GOOGLE_APPLICATION_CREDENTIALS # Provide a service account key file for GCS and BigQuery API calls (overrides application default credentials)
# export ZEPPELIN_NOTEBOOK_MONGO_URI # MongoDB connection URI used to connect to a MongoDB database server. Default "mongodb://localhost"
# export ZEPPELIN_NOTEBOOK_MONGO_DATABASE # Database name to store notebook. Default "zeppelin"
# export ZEPPELIN_NOTEBOOK_MONGO_COLLECTION # Collection name to store notebook. Default "notes"
# export ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT # If "true" import local notes under ZEPPELIN_NOTEBOOK_DIR on startup. Default "false"
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading
# export ZEPPELIN_HELIUM_NODE_INSTALLER_URL # Remote Node installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_NPM_INSTALLER_URL # Remote Npm installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL # Remote Yarn package installer url for Helium dependency loader
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise
#### Spark interpreter configuration ####
## Kerberos ticket refresh setting
##
#export KINIT_FAIL_THRESHOLD # (optional) How many times should kinit retry. The default value is 5.
#export KERBEROS_REFRESH_INTERVAL # (optional) The refresh interval for Kerberos ticket. The default value is 1d.
## Use provided spark installation ##
## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit
##

254
conf/zeppelin-site.xml.template
View file

@ -67,6 +67,29 @@
<description>hide homescreen notebook from list when this value set to true</description>
</property>
<property>
<name>zeppelin.notebook.collaborative.mode.enable</name>
<value>true</value>
<description>Enable collaborative mode</description>
</property>
<!-- Google Cloud Storage notebook storage -->
<!--
<property>
<name>zeppelin.notebook.gcs.dir</name>
<value></value>
<description>
A GCS path in the form gs://bucketname/path/to/dir.
Notes are stored at {zeppelin.notebook.gcs.dir}/{notebook-id}/note.json
</description>
</property>
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
-->
<!-- Amazon S3 notebook storage -->
<!-- Creates the following directory structure: s3://{bucket}/{username}/{notebook-id}/note.json -->
@ -138,6 +161,16 @@
</property>
-->
<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
<!--
<property>
<name>zeppelin.notebook.s3.signerOverride</name>
<value>S3SignerType</value>
<description>optional override to control which signature algorithm should be used to sign AWS requests</description>
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>
@ -173,6 +206,26 @@
</property>
-->
<!-- Notebook storage layer using hadoop compatible file system
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
<description>Hadoop compatible file system notebook persistence layer implementation, such as local file system, hdfs, azure wasb, s3 and etc.</description>
</property>
<property>
<name>zeppelin.server.kerberos.keytab</name>
<value></value>
<description>keytab for accessing kerberized hdfs</description>
</property>
<property>
<name>zeppelin.server.kerberos.principal</name>
<value></value>
<description>principal for accessing kerberized hdfs</description>
</property>
-->
<!-- For connecting your Zeppelin with ZeppelinHub -->
<!--
<property>
@ -182,6 +235,39 @@
</property>
-->
<!-- MongoDB notebook storage -->
<!--
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.MongoNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
<property>
<name>zeppelin.notebook.mongo.uri</name>
<value>mongodb://localhost</value>
<description>MongoDB connection URI used to connect to a MongoDB database server</description>
</property>
<property>
<name>zeppelin.notebook.mongo.database</name>
<value>zeppelin</value>
<description>database name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.mongo.collection</name>
<value>notes</value>
<description>collection name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.mongo.autoimport</name>
<value>false</value>
<description>import local notes into MongoDB automatically on startup</description>
</property>
-->
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
@ -206,21 +292,51 @@
<description>Local repository for interpreter's additional dependency loading</description>
</property>
<property>
<name>zeppelin.interpreter.dep.mvnRepo</name>
<value>http://repo1.maven.org/maven2/</value>
<description>Remote principal repository for interpreter's additional dependency loading</description>
</property>
<property>
<name>zeppelin.dep.localrepo</name>
<value>local-repo</value>
<description>Local repository for dependency loader</description>
</property>
<property>
<name>zeppelin.helium.node.installer.url</name>
<value>https://nodejs.org/dist/</value>
<description>Remote Node installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.npm.installer.url</name>
<value>http://registry.npmjs.org/</value>
<description>Remote Npm installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.yarnpkg.installer.url</name>
<value>https://github.com/yarnpkg/yarn/releases/download/</value>
<description>Remote Yarn package installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
<property>
<name>zeppelin.interpreter.group.order</name>
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy</value>
<description></description>
</property>
<property>
<name>zeppelin.interpreter.connect.timeout</name>
<value>30000</value>
<value>60000</value>
<description>Interpreter process connect timeout in msec.</description>
</property>
@ -300,6 +416,18 @@
<description>Anonymous user allowed by default</description>
</property>
<property>
<name>zeppelin.username.force.lowercase</name>
<value>false</value>
<description>Force convert username case to lower case, useful for Active Directory/LDAP. Default is not to change case</description>
</property>
<property>
<name>zeppelin.notebook.default.owner.username</name>
<value></value>
<description>Set owner role by default</description>
</property>
<property>
<name>zeppelin.notebook.public</name>
<value>true</value>
@ -318,4 +446,124 @@
<description>Enable directory listings on server.</description>
</property>
<!--
<property>
<name>zeppelin.interpreter.lifecyclemanager.class</name>
<value>org.apache.zeppelin.interpreter.lifecycle.TimeoutLifecycleManager</value>
<description>LifecycleManager class for managing the lifecycle of interpreters, by default interpreter will
be closed after timeout</description>
</property>
<property>
<name>zeppelin.interpreter.lifecyclemanager.timeout.checkinterval</name>
<value>60000</value>
<description>Milliseconds of the interval to checking whether interpreter is time out</description>
</property>
<property>
<name>zeppelin.interpreter.lifecyclemanager.timeout.threshold</name>
<value>3600000</value>
<description>Milliseconds of the interpreter timeout threshold, by default it is 1 hour</description>
</property>
-->
<!--
<property>
<name>zeppelin.server.jetty.name</name>
<value>Jetty(7.6.0.v20120127)</value>
<description>Hardcoding Application Server name to Prevent Fingerprinting</description>
</property>
-->
<!--
<property>
<name>zeppelin.server.jetty.request.header.size</name>
<value>8192</value>
<description>Http Request Header Size Limit (to prevent HTTP 413)</description>
</property>
-->
<!--
<property>
<name>zeppelin.server.xframe.options</name>
<value>SAMEORIGIN</value>
<description>The X-Frame-Options HTTP response header can be used to indicate whether or not a browser should be allowed to render a page in a frame/iframe/object.</description>
</property>
-->
<!--
<property>
<name>zeppelin.server.strict.transport</name>
<value>max-age=631138519</value>
<description>The HTTP Strict-Transport-Security response header is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Enable this when Zeppelin is running on HTTPS. Value is in Seconds, the default value is equivalent to 20 years.</description>
</property>
-->
<!--
<property>
<name>zeppelin.server.xxss.protection</name>
<value>1</value>
<description>The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. When value is set to 1 and a cross-site scripting attack is detected, the browser will sanitize the page (remove the unsafe parts).</description>
</property>
-->
<!--
<property>
<name>zeppelin.interpreter.callback.portRange</name>
<value>10000:10010</value>
</property>
-->
<!--
<property>
<name>zeppelin.recovery.storage.class</name>
<value>org.apache.zeppelin.interpreter.recovery.FileSystemRecoveryStorage</value>
<description>ReoveryStorage implementation</description>
</property>
-->
<!--
<property>
<name>zeppelin.recovery.dir</name>
<value>recovery</value>
<description>Location where recovery metadata is stored</description>
</property>
-->
<!-- GitHub configurations
<property>
<name>zeppelin.notebook.git.remote.url</name>
<value></value>
<description>remote Git repository URL</description>
</property>
<property>
<name>zeppelin.notebook.git.remote.username</name>
<value>token</value>
<description>remote Git repository username</description>
</property>
<property>
<name>zeppelin.notebook.git.remote.access-token</name>
<value></value>
<description>remote Git repository password</description>
</property>
<property>
<name>zeppelin.notebook.git.remote.origin</name>
<value>origin</value>
<description>Git repository remote</description>
</property>
<property>
<name>zeppelin.notebook.cron.enable</name>
<value>false</value>
<description>Notebook enable cron scheduler feature</description>
</property>
<property>
<name>zeppelin.notebook.cron.folders</name>
<value></value>
<description>Notebook cron folders</description>
</property>
-->
</configuration>

24
dev/change_scala_version.sh
View file

@ -34,7 +34,7 @@ if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then
usage
fi
TO_VERSION=$1
TO_VERSION="$1"
check_scala_version() {
for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
@ -42,12 +42,14 @@ check_scala_version() {
exit 1
}
check_scala_version "$TO_VERSION"
check_scala_version "${TO_VERSION}"
if [ $TO_VERSION = "2.11" ]; then
if [ "${TO_VERSION}" = "2.11" ]; then
FROM_VERSION="2.10"
SCALA_LIB_VERSION="2.11.7"
else
FROM_VERSION="2.11"
SCALA_LIB_VERSION="2.10.5"
fi
sed_i() {
@ -57,11 +59,17 @@ sed_i() {
export -f sed_i
BASEDIR=$(dirname $0)/..
find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \;
find "${BASEDIR}" -name 'pom.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(artifactId.*\)_'${FROM_VERSION}'/\1_'${TO_VERSION}'/g' {}" \;
# Also update <scala.binary.version> in parent POM
# update <scala.binary.version> in parent POM
# Match any scala binary version to ensure idempotency
sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'$TO_VERSION'</' \
"$BASEDIR/pom.xml"
sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'${TO_VERSION}'</' \
"${BASEDIR}/pom.xml"
# update <scala.version> in parent POM
# This is to make variables in leaf pom to be substituted to real value when flattened-pom is created.
# maven-flatten plugin doesn't take properties defined under profile even if scala-2.11/scala-2.10 is activated via -Pscala-2.11/-Pscala-2.10,
# and use default defined properties to create flatten pom.
sed_i '1,/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</s/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</<scala.version>'${SCALA_LIB_VERSION}'</' \
"${BASEDIR}/pom.xml"

80
dev/change_zeppelin_version.sh Executable file
View file

@ -0,0 +1,80 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
usage() {
echo "usage) $0 [OLD version] [NEW version]"
echo " ex. $0 0.9.0-SNAPSHOT 0.9.0"
exit 1
}
if [[ $# -ne 2 ]]; then
usage
fi
FROM_VERSION="$1"
TO_VERSION="$2"
is_dev_version() {
if [[ "$1" == *"SNAPSHOT" ]]; then
return 0
else
return 1
fi
}
is_maintenance_version() {
local version="$1"
if [[ "${version}" == *"SNAPSHOT" ]]; then
version = $(echo ${1} | cut -d'-' -f 1)
fi
if [[ "${version}" == *".0" ]]; then
return 1
else
return 0
fi
}
# Change version in pom.xml
mvn versions:set -DnewVersion="${TO_VERSION}" -DgenerateBackupPoms=false > /dev/null 2>&1
# Change version in example and package files
sed -i '' 's/-'"${FROM_VERSION}"'.jar",/-'"${TO_VERSION}"'.jar",/g' zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json
sed -i '' 's/"version": "'"${FROM_VERSION}"'",/"version": "'"${TO_VERSION}"'",/g' zeppelin-web/src/app/tabledata/package.json
sed -i '' 's/"version": "'"${FROM_VERSION}"'",/"version": "'"${TO_VERSION}"'",/g' zeppelin-web/src/app/visualization/package.json
sed -i '' 's/"version": "'"${FROM_VERSION}"'",/"version": "'"${TO_VERSION}"'",/g' zeppelin-web/src/app/spell/package.json
# Change version in Dockerfile
sed -i '' 's/Z_VERSION="'"${FROM_VERSION}"'"/Z_VERSION="'"${TO_VERSION}"'"/g' scripts/docker/zeppelin/bin/Dockerfile
# When preparing new dev version from release tag, doesn't need to change docs version
if is_dev_version "${FROM_VERSION}" || ! is_dev_version "${TO_VERSION}"; then
# When prepare new rc for the maintenance release
if is_dev_version "${FROM_VERSION}" && is_maintenance_version "${TO_VERSION}" \
&& [[ "${FROM_VERSION}" == "${TO_VERSION}"* ]]; then
FROM_VERSION=$(echo "${TO_VERSION}" | awk -F. '{ printf("%d.%d.%d", $1, $2, $3-1) }')
fi
# Change zeppelin version in docs config
sed -i '' 's/ZEPPELIN_VERSION : '"${FROM_VERSION}"'$/ZEPPELIN_VERSION : '"$TO_VERSION"'/g' docs/_config.yml
sed -i '' 's/BASE_PATH : \/docs\/'"${FROM_VERSION}"'$/BASE_PATH : \/docs\/'"$TO_VERSION"'/g' docs/_config.yml
# Change interpreter's maven version in docs and interpreter-list
sed -i '' 's/:'"${FROM_VERSION}"'/:'"${TO_VERSION}"'/g' conf/interpreter-list
sed -i '' 's/:'"${FROM_VERSION}"'/:'"${TO_VERSION}"'/g' docs/usage/interpreter/installation.md
fi

3
dev/common_release.sh
View file

@ -33,6 +33,9 @@ fi
mkdir "${WORKING_DIR}"
# If set to 'yes', release script will deploy artifacts to SNAPSHOT repository.
DO_SNAPSHOT='no'
usage() {
echo "usage) $0 [Release version] [Branch or Tag]"
echo " ex. $0 0.6.0 v0.6.0"

4
dev/create_release.sh
View file

@ -106,8 +106,8 @@ function make_binary_release() {
git_clone
make_source_package
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}"
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
# remove non release files and dirs
rm -rf "${WORKING_DIR}/zeppelin"

46
dev/publish_release.sh
View file

@ -1,5 +1,4 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@ -43,8 +42,11 @@ NC='\033[0m' # No Color
RELEASE_VERSION="$1"
GIT_TAG="$2"
if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
DO_SNAPSHOT="yes"
fi
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"
@ -67,6 +69,40 @@ function curl_error() {
fi
}
#
# Publishing Apache Zeppelin artifact to Apache snapshot repository.
#
function publish_snapshot_to_maven() {
cd "${WORKING_DIR}/zeppelin"
echo "Deploying Apache Zeppelin $RELEASE_VERSION version to snapshot repository."
if [[ ! $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
echo "ERROR: Snapshots must have a version containing 'SNAPSHOT'"
echo "ERROR: You gave version '$RELEASE_VERSION'"
exit 1
fi
tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)"
mvn versions:set -DnewVersion=$RELEASE_VERSION
tmp_settings="tmp-settings.xml"
echo "<settings><servers><server>" > $tmp_settings
echo "<id>apache.snapshots.https</id><username>$ASF_USERID</username>" >> $tmp_settings
echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
echo "</server></servers></settings>" >> $tmp_settings
mvn --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \
$PUBLISH_PROFILES -Drat.skip=true deploy
"${BASEDIR}/change_scala_version.sh" 2.11
mvn -Pscala-2.11 --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \
$PUBLISH_PROFILES -Drat.skip=true clean deploy
rm $tmp_settings
rm -rf $tmp_repo
}
function publish_to_maven() {
cd "${WORKING_DIR}/zeppelin"
@ -153,5 +189,9 @@ function publish_to_maven() {
}
git_clone
publish_to_maven
if [[ "${DO_SNAPSHOT}" == 'yes' ]]; then
publish_snapshot_to_maven
else
publish_to_maven
fi
cleanup

50
dev/tag_release.sh Executable file
View file

@ -0,0 +1,50 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
WORKING_DIR="/tmp/apache-zeppelin"
for var in CURRENT_VERSION RELEASE_VERSION NEXT_DEV_VERSION RC_TAG GIT_BRANCH; do
if [[ -z "${!var}" ]]; then
echo "You need ${var} variable set"
exit 1
fi
done
set -e
git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git "${WORKING_DIR}"
pushd "${WORKING_DIR}"
git checkout "${GIT_BRANCH}"
# Create release version
./dev/change_zeppelin_version.sh "${CURRENT_VERSION}" "${RELEASE_VERSION}"
git commit -a -m "Preparing Apache Zeppelin release ${RELEASE_VERSION}"
echo "Creating tag ${RC_TAG} at the head of ${GIT_BRANCH}"
git tag "${RC_TAG}"
# Create next dev version
./dev/change_zeppelin_version.sh "${RELEASE_VERSION}" "${NEXT_DEV_VERSION}"
git commit -a -m "Preparing development version ${NEXT_DEV_VERSION}"
git push origin "${RC_TAG}"
git push origin HEAD:"${GIT_BRANCH}"
popd
rm -rf "${WORKING_DIR}"

4
docs/README.md
View file

@ -56,8 +56,10 @@ If you wish to help us and contribute to Zeppelin Documentation, please look at
```
2. checkout ASF repo
```
svn co https://svn.apache.org/repos/asf/zeppelin asf-zeppelin
```
3. copy `zeppelin/docs/_site` to `asf-zeppelin/site/docs/[VERSION]`
4. ```svn commit```
4. `svn commit`

4
docs/_config.yml
View file

@ -21,7 +21,7 @@ author :
twitter : ASF
feedburner : feedname
ZEPPELIN_VERSION : 0.8.0-SNAPSHOT
ZEPPELIN_VERSION : 0.9.0-SNAPSHOT
# The production_url is only used when full-domain names are needed
# such as sitemap.txt
@ -59,7 +59,7 @@ JB :
# - Only the following values are falsy: ["", null, false]
# - When setting BASE_PATH it must be a valid url.
# This means always setting the protocol (http|https) or prefixing with "/"
BASE_PATH : /docs/0.8.0-SNAPSHOT
BASE_PATH : /docs/0.9.0-SNAPSHOT
# By default, the asset_path is automatically defined relative to BASE_PATH plus the enabled theme.
# ex: [BASE_PATH]/assets/themes/[THEME-NAME]

192
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -1,5 +1,5 @@
<div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation">
<div class="container">
<div class="container navbar-container">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="sr-only">Toggle navigation</span>
@ -9,11 +9,12 @@
</button>
<div class="navbar-brand">
<a class="navbar-brand-main" href="{{site.production_url}}">
<img src="/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" alt="I'm zeppelin">
<span style="vertical-align:middle">Zeppelin</span>
</a>
<a class="navbar-brand-version" href="{{BASE_PATH}}">
<span><small>{{site.ZEPPELIN_VERSION}}</small></span>
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/zeppelin_logo.png" width="50"
style="margin-top: -2px;" alt="I'm zeppelin">
<span style="margin-left: 5px; font-size: 27px;">Zeppelin</span>
<a class="navbar-brand-version" href="{{BASE_PATH}}"
style="font-size: 15px; color: white;"> {{site.ZEPPELIN_VERSION}}
</a>
</a>
</div>
</div>
@ -22,38 +23,107 @@
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="{{BASE_PATH}}/index.html">What is Apache Zeppelin ?</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Getting Started</b><span></li>
<li><a href="{{BASE_PATH}}/install/install.html">Install</a></li>
<li><a href="{{BASE_PATH}}/install/configuration.html">Configuration</a></li>
<li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li>
<li class="title"><span>Getting Started</span></li>
<li><a href="{{BASE_PATH}}/quickstart/install.html">Install</a></li>
<li><a href="{{BASE_PATH}}/quickstart/explore_ui.html">Explore UI</a></li>
<li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Basic Feature Guide</b><span></li>
<li><a href="{{BASE_PATH}}/manual/dynamicform.html">Dynamic Form</a></li>
<li><a href="{{BASE_PATH}}/manual/publish.html">Publish your Paragraph</a></li>
<li><a href="{{BASE_PATH}}/manual/notebookashomepage.html">Customize Zeppelin Homepage</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>More</b><span></li>
<li><a href="{{BASE_PATH}}/install/upgrade.html">Upgrade Zeppelin Version</a></li>
<li><a href="{{BASE_PATH}}/install/build.html">Build from source</a></li>
<li><a href="{{BASE_PATH}}/quickstart/install_with_flink_and_spark_cluster.html">Install Zeppelin with Flink and Spark Clusters Tutorial</a></li>
<li><a href="{{BASE_PATH}}/quickstart/spark_with_zeppelin.html">Spark with Zeppelin</a></li>
<li><a href="{{BASE_PATH}}/quickstart/sql_with_zeppelin.html">SQL with Zeppelin</a></li>
<li><a href="{{BASE_PATH}}/quickstart/python_with_zeppelin.html">Python with Zeppelin</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Usage<b class="caret"></b></a>
<ul class="dropdown-menu scrollable-menu">
<li class="title"><span>Dynamic Form</span></li>
<li><a href="{{BASE_PATH}}/usage/dynamic_form/intro.html">What is Dynamic Form?</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Display System</span></li>
<li><a href="{{BASE_PATH}}/usage/display_system/basic.html#text">Text Display</a></li>
<li><a href="{{BASE_PATH}}/usage/display_system/basic.html#html">HTML Display</a></li>
<li><a href="{{BASE_PATH}}/usage/display_system/basic.html#table">Table Display</a></li>
<li><a href="{{BASE_PATH}}/usage/display_system/basic.html#network">Network Display</a></li>
<li><a href="{{BASE_PATH}}/usage/display_system/angular_backend.html">Angular Display using Backend API</a></li>
<li><a href="{{BASE_PATH}}/usage/display_system/angular_frontend.html">Angular Display using Frontend API</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Interpreter</span></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/overview.html">Overview</a></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/interpreter_binding_mode.html">Interpreter Binding Mode</a></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/user_impersonation.html">User Impersonation</a></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/dependency_management.html">Dependency Management</a></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/installation.html">Installing Interpreters</a></li>
<!--<li><a href="{{BASE_PATH}}/usage/interpreter/dynamic_loading.html">Dynamic Interpreter Loading (Experimental)</a></li>-->
<li><a href="{{BASE_PATH}}/usage/interpreter/execution_hooks.html">Execution Hooks (Experimental)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Other Features</span></li>
<li><a href="{{BASE_PATH}}/usage/other_features/publishing_paragraphs.html">Publishing Paragraphs</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/personalized_mode.html">Personalized Mode</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/customizing_homepage.html">Customizing Zeppelin Homepage</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/notebook_actions.html">Notebook Actions</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/cron_scheduler.html">Cron Scheduler</a></li>
<li><a href="{{BASE_PATH}}/usage/other_features/zeppelin_context.html">Zeppelin Context</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>REST API</span></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/interpreter.html">Interpreter API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/zeppelin_server.html">Zeppelin Server API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/notebook.html">Notebook API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/notebook_repository.html">Notebook Repository API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/configuration.html">Configuration API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/credential.html">Credential API</a></li>
<li><a href="{{BASE_PATH}}/usage/rest_api/helium.html">Helium API</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Setup<b class="caret"></b></a>
<ul class="dropdown-menu scrollable-menu">
<li class="title"><span>Basics</span></li>
<li><a href="{{BASE_PATH}}/setup/basics/how_to_build.html">How to Build Zeppelin</a></li>
<li><a href="{{BASE_PATH}}/setup/basics/multi_user_support.html">Multi-user Support</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Deployment</span></li>
<!--<li><a href="{{BASE_PATH}}/setup/deployment/docker.html">Docker Image for Zeppelin</a></li>-->
<li><a href="{{BASE_PATH}}/setup/deployment/spark_cluster_mode.html#spark-standalone-mode">Spark Cluster Mode: Standalone</a></li>
<li><a href="{{BASE_PATH}}/setup/deployment/spark_cluster_mode.html#spark-on-yarn-mode">Spark Cluster Mode: YARN</a></li>
<li><a href="{{BASE_PATH}}/setup/deployment/spark_cluster_mode.html#spark-on-mesos-mode">Spark Cluster Mode: Mesos</a></li>
<li><a href="{{BASE_PATH}}/setup/deployment/flink_and_spark_cluster.html">Zeppelin with Flink, Spark Cluster</a></li>
<li><a href="{{BASE_PATH}}/setup/deployment/cdh.html">Zeppelin on CDH</a></li>
<li><a href="{{BASE_PATH}}/setup/deployment/virtual_machine.html">Zeppelin on VM: Vagrant</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Security</span></li>
<li><a href="{{BASE_PATH}}/setup/security/authentication_nginx.html">HTTP Basic Auth using NGINX</a></li>
<li><a href="{{BASE_PATH}}/setup/security/shiro_authentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/setup/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/setup/security/datasource_authorization.html">Data Source Authorization</a></li>
<li><a href="{{BASE_PATH}}/setup/security/http_security_headers.html">HTTP Security Headers</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Notebook Storage</span></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-zeppelinhub">ZeppelinHub Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-mongodb">MongoDB Storage</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span>Operation</span></li>
<li><a href="{{BASE_PATH}}/setup/operation/configuration.html">Configuration</a></li>
<li><a href="{{BASE_PATH}}/setup/operation/proxy_setting.html">Proxy Setting</a></li>
<li><a href="{{BASE_PATH}}/setup/operation/upgrading.html">Upgrading</a></li>
<li><a href="{{BASE_PATH}}/setup/operation/trouble_shooting.html">Trouble Shooting</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
<ul class="dropdown-menu scrollable-menu">
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li class="title"><span>Interpreters</span></li>
<li><a href="{{BASE_PATH}}/usage/interpreter/overview.html">Overview</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Usage</b><span></li>
<li><a href="{{BASE_PATH}}/manual/interpreterinstallation.html">Interpreter Installation</a></li>
<!--<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>-->
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
<li><a href="{{BASE_PATH}}/manual/userimpersonation.html">Interpreter User Impersonation</a></li>
<li><a href="{{BASE_PATH}}/manual/interpreterexechooks.html">Interpreter Execution Hooks (Experimental)</a></li>
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/python.html">Python</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Available Interpreters</b><span></li>
<li><a href="{{BASE_PATH}}/interpreter/alluxio.html">Alluxio</a></li>
<li><a href="{{BASE_PATH}}/interpreter/beam.html">Beam</a></li>
<li><a href="{{BASE_PATH}}/interpreter/bigquery.html">BigQuery</a></li>
@ -61,73 +131,47 @@
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
<li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
<li><a href="{{BASE_PATH}}/interpreter/geode.html">Geode</a></li>
<li><a href="{{BASE_PATH}}/interpreter/groovy.html">Groovy</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hdfs.html">HDFS</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/kylin.html">Kylin</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/neo4j.html">Neo4j</a></li>
<li><a href="{{BASE_PATH}}/interpreter/pig.html">Pig</a></li>
<li><a href="{{BASE_PATH}}/interpreter/python.html">Python</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, HAWQ</a></li>
<li><a href="{{BASE_PATH}}/interpreter/r.html">R</a></li>
<li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
<li><a href="{{BASE_PATH}}/interpreter/scio.html">Scio</a></li>
<li><a href="{{BASE_PATH}}/interpreter/shell.html">Shell</a></li>
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Display System <b class="caret"></b></a>
<ul class="dropdown-menu">
<li class="title"><span><b>Basic Display System</b><span></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#text">Text</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#html">Html</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table">Table</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Angular API</b><span></li>
<li><a href="{{BASE_PATH}}/displaysystem/back-end-angular.html">Angular (backend API)</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/front-end-angular.html">Angular (frontend API)</a></li>
</ul>
</li>
<li>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a>
<ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;">
<li class="title"><span><b>Notebook Storage</b><span></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#storage-in-zeppelinhub">ZeppelinHub Storage</a></li>
<li class="title"><span>Extending Zeppelin</span></li>
<li><a href="{{BASE_PATH}}/development/writing_zeppelin_interpreter.html">Writing Zeppelin Interpreter</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>REST API</b><span></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-interpreter.html">Interpreter API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-notebook.html">Notebook API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-configuration.html">Configuration API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-credential.html">Credential API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-helium.html">Helium API</a></li>
<li class="title"><span>Helium (Experimental)</span></li>
<li><a href="{{BASE_PATH}}/development/helium/overview.html">Overview</a></li>
<li><a href="{{BASE_PATH}}/development/helium/writing_application.html">Writing Helium Application</a></li>
<li><a href="{{BASE_PATH}}/development/helium/writing_spell.html">Writing Helium Spell</a></li>
<li><a href="{{BASE_PATH}}/development/helium/writing_visualization_basic.html">Writing Helium Visualization: Basics</a></li>
<li><a href="{{BASE_PATH}}/development/helium/writing_visualization_transformation.html">Writing Helium Visualization: Transformation</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Security</b><span></li>
<li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
<li class="title"><span>Contributing to Zeppelin</span></li>
<li><a href="{{BASE_PATH}}/setup/basics/how_to_build.html">How to Build Zeppelin</a></li>
<li><a href="{{BASE_PATH}}/development/contribution/useful_developer_tools.html">Useful Developer Tools</a></li>
<li><a href="{{BASE_PATH}}/development/contribution/how_to_contribute_code.html">How to Contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/contribution/how_to_contribute_website.html">How to Contribute (website)</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Advanced</b><span></li>
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li>
<li><a href="{{BASE_PATH}}/install/cdh.html">Zeppelin on CDH</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Contibute</b><span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell (Experimental)</a></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li class="title"><span>External Resources</span></li>
<li><a target="_blank" href="https://zeppelin.apache.org/community.html">Mailing List</a></li>
<li><a target="_blank" href="https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home">Apache Zeppelin Wiki</a></li>
<li><a target="_blank" href="http://stackoverflow.com/questions/tagged/apache-zeppelin">Stackoverflow Questions about Zeppelin</a></li>
</ul>
</li>
<li>

145
docs/assets/themes/zeppelin/css/style.css
View file

@ -1,14 +1,14 @@
/* Move down content because we have a fixed navbar that is 50px tall */
/* Move down content because we have a fixed navbar that is 60px tall */
@import url(//fonts.googleapis.com/css?family=Patua+One);
@import url(//fonts.googleapis.com/css?family=Open+Sans);
body {
padding-top: 50px;
padding-bottom: 20px;
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
line-height: 1.6;
color: #4c555a;
letter-spacing: .2px;
font-size: 15px;
}
.jumbotron {
@ -58,6 +58,7 @@ body {
.jumbotron h1 {
font-family: 'Patua One', cursive;
}
.jumbotron h1 {
font-family: 'Patua One', cursive;
@ -67,7 +68,6 @@ body {
font-size: 60%;
color: #FFF;
}
}
.jumbotron small {
font-size: 60%;
@ -75,8 +75,8 @@ body {
}
.navbar-brand {
padding-top: 10px;
padding-bottom: 10px;
padding-top: 15px;
padding-bottom: 15px;
}
.navbar-brand-main img {
@ -86,16 +86,20 @@ body {
.navbar {
background: #3071a9;
border-bottom: 0px;
height: 50px;
height: 60px;
box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.4);
}
.navbar-inverse .navbar-nav > li > a {
padding-top: 18px;
padding-bottom: 20px;
font-size: 15px;
font-weight: 300;
color: white;
background: #3071a9;
}
.navbar-inverse .navbar-nav > li > a:hover,
.navbar-inverse .navbar-nav > li > a:focus {
.navbar-inverse .navbar-nav > li > a:hover {
color: white !important;
background: #2C6094 !important;
}
@ -181,7 +185,12 @@ body {
}
.navbar-inverse .navbar-toggle {
border-color: #265380;
margin-top: 12px;
border-color: transparent;
}
.navbar-inverse .navbar-toggle:focus {
outline-width: 0;
}
.navbar-inverse .navbar-toggle:hover,
@ -190,8 +199,9 @@ body {
background: #265380;
}
.navbar-inverse .navbar-toggle:focus {
outline-width: 0;
.navbar-inverse .navbar-toggle.collapsed {
border-color: #3071A9;
background: #3071a9;
}
/* CUSTOMIZE THE CAROUSEL
@ -257,8 +267,9 @@ body {
.content {
word-wrap: break-word;
}
.content :first-child {
margin-top: 0;
margin-top: 20px;
}
@media screen and (min-width: 64em) {
@ -299,7 +310,7 @@ a.anchor {
/* <hn> */
.content h1, h2, h3, h4, h5, h6 {
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-family: "Roboto", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
margin-top: 3rem;
margin-bottom: 1rem;
font-weight: bold;
@ -365,7 +376,7 @@ a.anchor {
padding: 0.8rem;
margin-top: 0;
margin-bottom: 1rem;
font: Consolas, "Liberation Mono", Menlo, Courier, monospace;
font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
font-size: 90%;
color: #567482;
word-wrap: normal;
@ -537,7 +548,7 @@ a.anchor {
}
.navbar-collapse.collapse {
max-height: 50px;
max-height: 60px;
}
.navbar-inverse .navbar-nav a .caret,
@ -594,47 +605,26 @@ a.anchorjs-link:hover { text-decoration: none; }
padding-bottom: 3px;
}
/* Custom, iPhone Retina */
@media only screen and (max-width: 480px) {
.jumbotron h1 {
display: none;
}
.navbar-brand-version small {
display: none;
color: white;
}
}
@media only screen and (max-width: 768px) {
.navbar .navbar-brand-main {
padding-bottom: 0;
}
}
@media only screen
and (min-width: 768px)
and (max-width: 1024px) {
.navbar-brand-version small {
display: none;
}
.navbar-collapse.collapse {
padding-right: 0;
}
.navbar-fixed-top > .container {
width: 800px;
}
.navbar-inverse .navbar-brand {
color: white;
text-decoration: none;
font-size: 32px;
font-size: 27px;
}
/* master branch docs dropdown menu */
#menu .dropdown-menu li span {
padding: 3px 10px 10px 10px;
font-size: 13px;
font-family: "Roboto", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
padding: 12px 10px 12px 20px;
font-size: 16px;
font-weight: 300;
}
#menu .dropdown-menu li a {
padding-left: 30px;
}
#menu .title {
padding-bottom: 8px;
color: #3071a9;
}
@ -649,7 +639,7 @@ and (max-width: 1024px) {
}
#menu .navbar-brand-version {
margin-right: 50px;
margin-right: 30px;
text-decoration: none !important;
}
@ -659,9 +649,8 @@ and (max-width: 1024px) {
vertical-align: bottom;
}
#menu .navbar-brand-version small {
#menu .navbar-brand-version {
font-size: 14px;
font-family: 'Open Sans', 'Helvetica Neue', Helvetica;
color: white;
}
@ -676,14 +665,64 @@ and (max-width: 1024px) {
font-size: 13px;
}
/* Custom, iPhone Retina */
@media only screen and (max-width: 480px) {
.jumbotron h1 {
display: none;
}
.navbar-brand-version {
display: none;
color: white;
}
}
@media only screen and (max-width: 480px) {
#menu .title {
color: #bbb;
}
}
/* when navigation toggle is enabled due to browser size */
@media only screen and (max-width: 768px) {
#menu .title {
color: #bbb;
text-shadow: 1px 2px #353131;
color: white;
font-family: "Roboto", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 16px;
font-weight: 300;
}
.content {
padding-left: 30px;
padding-right: 30px;
}
.navbar .navbar-brand-main {
padding-bottom: 0;
}
.navbar-nav {
margin-top: 0px;
}
.navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form {
border-color: #73a0cd;
}
}
@media only screen and (min-width: 768px) and (max-width: 1024px) {
.navbar-brand-version {
display: none;
}
.navbar-collapse.collapse {
padding-right: 0;
}
.navbar-fixed-top > .container {
width: auto;
}
}

BIN
docs/assets/themes/zeppelin/img/available_interpreters.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 195 KiB

After

Width:  |  Height:  |  Size: 196 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/cron_scheduler_dialog_box.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 50 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-example-spark-isolated.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 474 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-example-spark-scoped.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 434 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-example-spark-shared.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 208 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-isolated.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 434 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-scoped.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 406 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-shared.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 194 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_mode-structure.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.3 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_per_note_user.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 66 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/interpreter_binding_scoped_isolated.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 67 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-config.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 21 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-graph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 29 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.5 MiB

BIN
docs/assets/themes/zeppelin/img/docs-img/revisions-comparator-button.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 19 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/revisions-comparator-comboboxes.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.5 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/revisions-comparator-diff.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 60 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/revisions-comparator-paragraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 65 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/revisions-comparator-table.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 16 KiB

BIN
docs/assets/themes/zeppelin/img/pig_zeppelin_tutorial.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 274 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/conf_interpreter.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 38 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 48 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 52 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 41 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/form_password.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 28 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/form_password_prog.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 34 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/interpreter_precode.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 8.1 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/interpreter_setting_with_context_parameters.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 17 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 9.2 KiB

BIN
docs/assets/themes/zeppelin/img/ui-img/about_menu.png Normal file → Executable file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 72 KiB

After

Width:  |  Height:  |  Size: 89 KiB

BIN
docs/assets/themes/zeppelin/img/ui-img/settings_menu.png Normal file → Executable file
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 36 KiB

9
docs/assets/themes/zeppelin/js/docs.js
View file

@ -122,6 +122,15 @@ $(function() {
$('#toc').toc();
});
$(document).click(function(event) {
// fold navbar when a user click other components
var clickover = $(event.target);
var _opened = $(".navbar-collapse").hasClass("navbar-collapse in");
if (_opened === true && !clickover.hasClass("navbar-toggle")) {
$("button.navbar-toggle").click();
}
});
// Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers
// will try to do *their* initial scroll after running the onReady handler.
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });

55
docs/development/howtocontribute.md → docs/development/contribution/how_to_contribute_code.md
View file

@ -2,7 +2,7 @@
layout: page
title: "Contributing to Apache Zeppelin (Code)"
description: "How can you contribute to Apache Zeppelin project? This document covers from setting up your develop environment to making a pull request on Github."
group: development
group: development/contribution
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -41,7 +41,7 @@ You are free to use whatever IDE you prefer, or your favorite command line edito
To build the code, install
* Oracle Java 7
* Oracle Java 8
* Apache Maven
## Getting the source code
@ -51,13 +51,13 @@ First of all, you need Zeppelin source code. The official location of Zeppelin i
Get the source code on your development machine using git.
```
```bash
git clone git://git.apache.org/zeppelin.git zeppelin
```
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
```bash
git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin
```
@ -69,19 +69,19 @@ Before making a pull request, please take a look [Contribution Guidelines](http:
### Build
```
```bash
mvn install
```
To skip test
```
```bash
mvn install -DskipTests
```
To build with specific spark / hadoop version
```
```bash
mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x
```
@ -89,29 +89,48 @@ For the further
### Run Zeppelin server in development mode
```
#### Option 1 - Command Line
1. Copy the `conf/zeppelin-site.xml.template` to `zeppelin-server/src/main/resources/zeppelin-site.xml` and change the configurations in this file if required
2. Run the following command
```bash
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME \
mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
> **Note:** Make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro.
#### Option 2 - Daemon Script
> **Note:** Make sure you first run
```bash
mvn clean install -DskipTests
```
in your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro.
or use daemon script
```
```bash
bin/zeppelin-daemon start
```
Server will be run on [http://localhost:8080](http://localhost:8080).
#### Option 3 - IDE
1. Copy the `conf/zeppelin-site.xml.template` to `zeppelin-server/src/main/resources/zeppelin-site.xml` and change the configurations in this file if required
2. `ZeppelinServer.java` Main class
### Generating Thrift Code
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
To regenerate the code, install **thrift-0.9.2** and then run the following command to generate thrift code.
```
```bash
cd <zeppelin_home>/zeppelin-interpreter/src/main/thrift
./genthrift.sh
```
@ -120,14 +139,16 @@ cd <zeppelin_home>/zeppelin-interpreter/src/main/thrift
Zeppelin has [set of integration tests](https://github.com/apache/zeppelin/tree/master/zeppelin-server/src/test/java/org/apache/zeppelin/integration) using Selenium. To run these test, first build and run Zeppelin and make sure Zeppelin is running on port 8080. Then you can run test using following command
```
TEST_SELENIUM=true mvn test -Dtest=[TEST_NAME] -DfailIfNoTests=false -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
```bash
TEST_SELENIUM=true mvn test -Dtest=[TEST_NAME] -DfailIfNoTests=false \
-pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
```
For example, to run [ParagraphActionIT](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java),
```
TEST_SELENIUM=true mvn test -Dtest=ParagraphActionsIT -DfailIfNoTests=false -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
```bash
TEST_SELENIUM=true mvn test -Dtest=ParagraphActionsIT -DfailIfNoTests=false \
-pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
```
You'll need Firefox web browser installed in your development environment. While CI server uses [Firefox 31.0](https://ftp.mozilla.org/pub/firefox/releases/31.0/) to run selenium test, it is good idea to install the same version (disable auto update to keep the version).

4
docs/development/howtocontributewebsite.md → docs/development/contribution/how_to_contribute_website.md
View file

@ -2,7 +2,7 @@
layout: page
title: "Contributing to Apache Zeppelin (Website)"
description: "How can you contribute to Apache Zeppelin project website? This document covers from building Zeppelin documentation site to making a pull request on Github."
group: development
group: development/contribution
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -39,7 +39,7 @@ Documentation website is hosted in 'master' branch under `/docs/` dir.
First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git).
Get the source code on your development machine using git.
```
```bash
git clone git://git.apache.org/zeppelin.git
cd docs
```

104
docs/development/contribution/useful_developer_tools.md Normal file
View file

@ -0,0 +1,104 @@
---
layout: page
title: "Useful Developer Tools"
description: ""
group: development/contribution
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Useful Developer Tools
<div id="toc"></div>
### Developing `zeppelin-web`
Check [zeppelin-web: Local Development](https://github.com/apache/zeppelin/tree/master/zeppelin-web#local-development).
### Tools
#### SVM: Scala Version Manager
[svm](https://github.com/yuroyoro/svm) would be useful when changing scala version frequently.
#### JDK change script: OSX
this script would be helpful when changing JDK version frequently.
```bash
function setjdk() {
if [ $# -ne 0 ]; then
# written based on OSX.
# use diffrent base path for other OS
removeFromPath '/System/Library/Frameworks/JavaVM.framework/Home/bin'
if [ -n "${JAVA_HOME+x}" ]; then
removeFromPath $JAVA_HOME
fi
export JAVA_HOME=`/usr/libexec/java_home -v $@`
export PATH=$JAVA_HOME/bin:$PATH
fi
}
function removeFromPath() {
export PATH=$(echo $PATH | sed -E -e "s;:$1;;" -e "s;$1:?;;")
}
```
you can use this function like `setjdk 1.8` / `setjdk 1.7`
### Building Submodules Selectively
```bash
# build `zeppelin-web` only
mvn clean -pl 'zeppelin-web' package -DskipTests;
# build `zeppelin-server` and its dependencies only
mvn clean package -pl 'spark,spark-dependencies,python,markdown,zeppelin-server' --am -DskipTests
# build spark related modules with default profiles: scala 2.10
mvn clean package -pl 'spark,spark-dependencies,zeppelin-server' --am -DskipTests
# build spark related modules with profiles: scala 2.11, spark 2.1 hadoop 2.7
./dev/change_scala_version.sh 2.11
mvn clean package -Pspark-2.1 -Phadoop-2.7 -Pscala-2.11 \
-pl 'spark,spark-dependencies,zeppelin-server' --am -DskipTests
# build `zeppelin-server` and `markdown` with dependencies
mvn clean package -pl 'markdown,zeppelin-server' --am -DskipTests
```
### Running Individual Tests
```bash
# run the `HeliumBundleFactoryTest` test class
mvn test -pl 'zeppelin-server' --am -DfailIfNoTests=false -Dtest=HeliumBundleFactoryTest
```
### Running Selenium Tests
Make sure that Zeppelin instance is started to execute integration tests (= selenium tests).
```bash
# run the `SparkParagraphIT` test class
TEST_SELENIUM="true" mvn test -pl 'zeppelin-server' --am \
-DfailIfNoTests=false -Dtest=SparkParagraphIT
# run the `testSqlSpark` test function only in the `SparkParagraphIT` class
# but note that, some test might be dependent on the previous tests
TEST_SELENIUM="true" mvn test -pl 'zeppelin-server' --am \
-DfailIfNoTests=false -Dtest=SparkParagraphIT#testSqlSpark
```

39
docs/development/helium/overview.md Normal file
View file

@ -0,0 +1,39 @@
---
layout: page
title: "Helium"
description: ""
group: development/helium
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Helium Overview
<div id="toc"></div>
## What is Helium?
Helium is a plugin system that can extend Zeppelin a lot.
For example, you can write [custom display system](./writing_spell.html) or
install already published one in [Heliun Online Registry](http://zeppelin.apache.org/helium_packages.html).
Currently, Helium supports 4 types of package.
- [Helium Visualization](./writing_visualization_basic.html): Adding a new chart type
- [Helium Spell](./writing_spell.html): Adding new interpreter, display system running on browser
- [Helium Application](./writing_application.html)
- [Helium Interpreter](../writing_zeppelin_interpreter.html): Adding a new custom interpreter

28
docs/development/writingzeppelinapplication.md → docs/development/helium/writing_application.md
View file

@ -1,8 +1,8 @@
---
layout: page
title: "Writing a new Application(Experimental)"
title: "Writing a new Helium Application"
description: "Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. Make your own Application in Apache Zeppelin is quite easy."
group: development
group: development/helium
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -19,7 +19,7 @@ limitations under the License.
-->
{% include JB/setup %}
# Writing a new Application (Experimental)
# Writing a new Application
<div id="toc"></div>
@ -91,19 +91,18 @@ In the Zeppelin notebook, run `%dev run` will connect to application running in
Package file is a json file that provides information about the application.
Json file contains the following information
```
```json
{
name : "[organization].[name]",
description : "Description",
artifact : "groupId:artifactId:version",
className : "your.package.name.YourApplicationClass",
resources : [
"name" : "[organization].[name]",
"description" : "Description",
"artifact" : "groupId:artifactId:version",
"className" : "your.package.name.YourApplicationClass",
"resources" : [
["resource.name", ":resource.class.name"],
["alternative.resource.name", ":alternative.class.name"]
],
icon : "<i class="icon"></i>"
"icon" : "<i class='icon'></i>"
}
```
#### name
@ -148,7 +147,7 @@ Resouce name is a string which will be compared with the name of objects in the
Application may require two or more resources. Required resources can be listed inside of the json array. For example, if the application requires object "name1", "name2" and "className1" type of object to run, resources field can be
```
```json
resources: [
[ "name1", "name2", ":className1", ...]
]
@ -156,7 +155,7 @@ resources: [
If Application can handle alternative combination of required resources, alternative set can be listed as below.
```
```json
resources: [
[ "name", ":className"],
[ "altName", ":altClassName1"],
@ -166,7 +165,7 @@ resources: [
Easier way to understand this scheme is
```
```json
resources: [
[ 'resource' AND 'resource' AND ... ] OR
[ 'resource' AND 'resource' AND ... ] OR
@ -184,4 +183,3 @@ e.g.
```
icon: "<i class='fa fa-clock-o'></i>"
```

56
docs/development/writingzeppelinspell.md → docs/development/helium/writing_spell.md
View file

@ -1,8 +1,8 @@
---
layout: page
title: "Writing a new Spell(Experimental)"
title: "Writing a new Helium Spell"
description: "Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter. "
group: development
group: development/helium
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -19,22 +19,22 @@ limitations under the License.
-->
{% include JB/setup %}
# Writing a new Spell (Experimental)
# Writing a new Spell
<div id="toc"></div>
## What is Apache Zeppelin Spell
## What is Apache Zeppelin Spell
Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter.
Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter.
It can provide many benefits.
- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry.
- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry.
- Every spell is written in javascript. It means you can use existing javascript libraries whatever you want.
- Spell runs on browser like display system (`%html`, `%table`). In other words, every spell can be used as display system as well.
## How it works
Helium Spell works like [Helium Visualization](./writingzeppelinvisualization.html).
Helium Spell works like [Helium Visualization](./writing_visualization_basic.html).
- Every helium packages are loaded from central (online) registry or local registry
- You can see loaded packages in `/helium` page.
@ -47,19 +47,19 @@ Helium Spell works like [Helium Visualization](./writingzeppelinvisualization.ht
Find a spell what you want to use in `/helium` package and click `Enable` button.
<img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_spell_registered.png" />
<img class="img-responsive" style="width:70%" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/writing_spell_registered.png" />
### 2. Using
Spell works like an interpreter. Use the `MAGIC` value to execute spell in a note. (you might need to refresh after enabling)
For example, Use `%echo` for the Echo Spell.
<img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_spell_using.gif" />
<img class="img-responsive" style="width:70%" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/writing_spell_using.gif" />
## Write a new Spell
## Write a new Spell
Making a new spell is similar to [Helium Visualization#write-new-visualization](./writingzeppelinvisualization.html#write-new-visualization).
Making a new spell is similar to [Helium Visualization#write-new-visualization](./writing_visualization_basic.html#write-new-visualization).
- Add framework dependency called zeppelin-spell into `package.json`
- Write code using framework
@ -67,14 +67,14 @@ Making a new spell is similar to [Helium Visualization#write-new-visualization](
### 1. Create a npm package
Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell.
Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell.
- You have to add a framework called `zeppelin-spell` as a dependency to create spell ([zeppelin-spell](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/spell))
- Also, you can add any dependencies you want to utilise.
Here's an example
```
```json
{
"name": "zeppelin-echo-spell",
"description": "Zeppelin Echo Spell (example)",
@ -95,7 +95,7 @@ Here's an example
}
```
### 2. Write spell using framework
### 2. Write spell using framework
Here are some examples you can refer
@ -106,7 +106,7 @@ Here are some examples you can refer
Now, you need to write code to create spell which processing text.
```javascript
```js
import {
SpellBase,
SpellResult,
@ -121,8 +121,8 @@ export default class EchoSpell extends SpellBase {
interpret(paragraphText) {
const processed = paragraphText + '!';
/**
/**
* should return `SpellResult` which including `data` and `type`
* default type is `TEXT` if you don't specify.
*/
@ -133,7 +133,7 @@ export default class EchoSpell extends SpellBase {
Here is another example. Let's say we want to create markdown spell. First of all, we should add a dependency for markdown in package.json
```
```json
// package.json
"dependencies": {
"markdown": "0.5.0",
@ -143,7 +143,7 @@ Here is another example. Let's say we want to create markdown spell. First of al
And here is spell code.
```javascript
```js
import {
SpellBase,
SpellResult,
@ -171,16 +171,16 @@ export default class MarkdownSpell extends SpellBase {
}
```
- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js)
- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js)
- You might want to return promise not string (e.g API call), then refer [Google Translation API Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-translator/index.js)
### 3. Create __Helium package file__ for local deployment
### 3. Create __Helium package file__ for local deployment
You don't want to publish your package every time you make a change in your spell. Zeppelin provides local deploy.
The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy.
The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy.
It's automatically created when you publish to npm repository but in local case, you should make it by yourself.
```
```json
{
"type" : "SPELL",
"name" : "zeppelin-echo-spell",
@ -197,18 +197,18 @@ It's automatically created when you publish to npm repository but in local case,
- Place this file in your local registry directory (default `$ZEPPELIN_HOME/helium`).
- `type` should be `SPELL`
- Make sure that `artifact` should be same as your spell directory.
- You can get information about other fields in [Helium Visualization#3-create-helium-package-file-and-locally-deploy](./writingzeppelinvisualization.html#3-create-helium-package-file-and-locally-deploy).
- Make sure that `artifact` should be same as your spell directory.
- You can get information about other fields in [Helium Visualization#3-create-helium-package-file-and-locally-deploy](./writing_visualization_basic.html#3-create-helium-package-file-and-locally-deploy).
### 4. Run in dev mode
```
```bash
cd zeppelin-web
yarn run dev:helium
yarn run dev:helium
```
You can browse localhost:9000. Every time refresh your browser, Zeppelin will rebuild your spell and reload changes.
### 5. Publish your spell to the npm repository
### 5. Publish your spell to the npm repository
See [Publishing npm packages](https://docs.npmjs.com/getting-started/publishing-npm-packages)

55
docs/development/writingzeppelinvisualization.md → docs/development/helium/writing_visualization_basic.md
View file

@ -1,8 +1,8 @@
---
layout: page
title: "Writing a new Visualization(Experimental)"
title: "Writing a new Helium Visualization: basic"
description: "Apache Zeppelin Visualization is a pluggable package that can be loaded/unloaded on runtime through Helium framework in Zeppelin. A Visualization is a javascript npm package and user can use them just like any other built-in visualization in a note."
group: development
group: development/helium
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -19,7 +19,7 @@ limitations under the License.
-->
{% include JB/setup %}
# Writing a new Visualization (Experimental)
# Writing a new Visualization
<div id="toc"></div>
@ -41,7 +41,7 @@ Once Zeppelin loads _Helium package files_ from registries, available packages a
Click 'enable' button.
<img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_visualization_helium_menu.png" />
<img class="img-responsive" style="width:70%" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/writing_visualization_helium_menu.png" />
#### 3. Create and load visualization bundle on the fly
@ -53,7 +53,7 @@ Once a Visualization package is enabled, [HeliumBundleFactory](https://github.co
Zeppelin shows additional button for loaded Visualizations.
User can use just like any other built-in visualizations.
<img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_visualization_example.png" />
<img class="img-responsive" style="width:70%" src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/writing_visualization_example.png" />
@ -61,11 +61,11 @@ User can use just like any other built-in visualizations.
#### 1. Create a npm package
Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. Normally, you can add any dependencies in package.json however Zeppelin Visualization package only allows two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).
Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. You can add any dependencies in package.json, but you **must include two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).**
Here's an example
```
```json
{
"name": "zeppelin_horizontalbar",
"description" : "Horizontal Bar chart",
@ -86,7 +86,7 @@ To create your own visualization, you need to create a js file and import [Visua
[Visualization](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/visualization/visualization.js) class, there're several methods that you need to override and implement. Here's simple visualization that just prints `Hello world`.
```
```js
import Visualization from 'zeppelin-vis'
import PassthroughTransformation from 'zeppelin-tabledata/passthrough'
@ -118,7 +118,7 @@ Zeppelin's built-in visualization uses the same API, so you can check [built-in
__Helium Package file__ is a json file that provides information about the application.
Json file contains the following information
```
```json
{
"type" : "VISUALIZATION",
"name" : "zeppelin_horizontalbar",
@ -134,8 +134,10 @@ Place this file in your local registry directory (default `./helium`).
##### type
When you're creating a visualization, 'type' should be 'VISUALIZATION'.
Check [application](./writingzeppelinapplication.html) type if you're interested in the other types of package.
When you're creating a visualization, 'type' should be 'VISUALIZATION'. Check these types as well.
- [Helium Application](./writing_application.html)
- [Helium Spell](./writing_spell.html)
##### name
@ -154,15 +156,15 @@ e.g.
When artifact exists in npm repository
```
artifact: "my-visualiztion@1.0.0"
```json
"artifact": "my-visualiztion@1.0.0"
```
When artifact exists in local file system
```
artifact: "/path/to/my/visualization"
```json
"artifact": "/path/to/my/visualization"
```
##### license
@ -171,8 +173,8 @@ License information.
e.g.
```
license: "Apache-2.0"
```json
"license": "Apache-2.0"
```
##### icon
@ -181,25 +183,30 @@ Icon to be used in visualization select button. String in this field will be ren
e.g.
```
icon: "<i class='fa fa-coffee'></i>"
```json
"icon": "<i class='fa fa-coffee'></i>"
```
#### 4. Run in dev mode
Place your __Helium package file__ in local registry (ZEPPELIN_HOME/helium).
Place your __Helium package file__ in local registry (`ZEPPELIN_HOME/helium`).
Run Zeppelin. And then run zeppelin-web in visualization dev mode.
```
```bash
cd zeppelin-web
yarn run dev:helium
yarn run dev:helium
```
You can browse localhost:9000. Everytime refresh your browser, Zeppelin will rebuild your visualization and reload changes.
You can browse `localhost:9000`. Everytime refresh your browser, Zeppelin will rebuild your visualization and reload changes.
#### 5. Publish your visualization
Once it's done, publish your visualization package using `npm publish`.
That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.
That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.
### See More
Check [Helium Visualization: Transformation](./writing_visualization_transformation.html) for more complex examples.

281
docs/development/helium/writing_visualization_transformation.md Normal file
View file

@ -0,0 +1,281 @@
---
layout: page
title: "Transformations in Zeppelin Visualization"
description: "Description for Transformations"
group: development/helium
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Transformations for Zeppelin Visualization
<div id="toc"></div>
## Overview
Transformations
- **renders** setting which allows users to set columns and
- **transforms** table rows according to the configured columns.
Zeppelin provides 4 types of transformations.
## 1. PassthroughTransformation
`PassthroughTransformation` is the simple transformation which does not convert original tabledata at all.
See [passthrough.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/passthrough.js)
## 2. ColumnselectorTransformation
`ColumnselectorTransformation` is uses when you need `N` axes but do not need aggregation.
See [columnselector.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/columnselector.js)
## 3. PivotTransformation
`PivotTransformation` provides group by and aggregation. Every chart using `PivotTransformation` has 3 axes. `Keys`, `Groups` and `Values`.
See [pivot.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/pivot.js)
## 4. AdvancedTransformation
`AdvancedTransformation` has more detailed options while providing existing features of `PivotTransformation` and `ColumnselectorTransformation`
- multiple sub charts
- configurable chart axes
- parameter widgets: `input`, `checkbox`, `option`, `textarea`
- parsing parameters automatically based on their types
- expand / fold axis and parameter panels
- multiple transformation methods while supporting lazy converting
- re-initialize the whole configuration based on spec hash.
### Spec
`AdvancedTransformation` requires `spec` which includes axis and parameter details for charts.
Let's create 2 sub-charts called `line` and `no-group`. Each sub chart can have different axis and parameter depending on their requirements.
<br/>
```js
class AwesomeVisualization extends Visualization {
constructor(targetEl, config) {
super(targetEl, config)
const spec = {
charts: {
'line': {
transform: { method: 'object', },
sharedAxis: false, /** set if you want to share axes between sub charts, default is `false` */
axis: {
'xAxis': { dimension: 'multiple', axisType: 'key', description: 'serial', },
'yAxis': { dimension: 'multiple', axisType: 'aggregator', description: 'serial', },
'category': { dimension: 'multiple', axisType: 'group', description: 'categorical', },
},
parameter: {
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
'lineWidth': { valueType: 'int', defaultValue: 0, description: 'width of line', },
},
},
'no-group': {
transform: { method: 'object', },
sharedAxis: false,
axis: {
'xAxis': { dimension: 'single', axisType: 'key', },
'yAxis': { dimension: 'multiple', axisType: 'value', },
},
parameter: {
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
},
},
}
this.transformation = new AdvancedTransformation(config, spec)
}
...
// `render` will be called whenever `axis` or `parameter` is changed
render(data) {
const { chart, parameter, column, transformer, } = data
if (chart === 'line') {
const transformed = transformer()
// draw line chart
} else if (chart === 'no-group') {
const transformed = transformer()
// draw no-group chart
}
}
}
```
<br/>
### Spec: `axis`
| Field Name | Available Values (type) | Description |
| --- | --- | --- |
|`dimension` | `single` | Axis can contains only 1 column |
|`dimension` | `multiple` | Axis can contains multiple columns |
|`axisType` | `key` | Column(s) in this axis will be used as `key` like in `PivotTransformation`. These columns will be served in `column.key` |
|`axisType` | `aggregator` | Column(s) in this axis will be used as `value` like in `PivotTransformation`. These columns will be served in `column.aggregator` |
|`axisType` | `group` | Column(s) in this axis will be used as `group` like in `PivotTransformation`. These columns will be served in `column.group` |
|`axisType` | (string) | Any string value can be used here. These columns will be served in `column.custom` |
|`maxAxisCount` (optional) | (int) | The max number of columns that this axis can contain. (unlimited if `undefined`) |
|`minAxisCount` (optional) | (int) | The min number of columns that this axis should contain to draw chart. (`1` in case of single dimension) |
|`description` (optional) | (string) | Description for the axis. |
<br/>
Here is an example.
```js
axis: {
'xAxis': { dimension: 'multiple', axisType: 'key', },
'yAxis': { dimension: 'multiple', axisType: 'aggregator'},
'category': { dimension: 'multiple', axisType: 'group', maxAxisCount: 2, valueType: 'string', },
},
```
<br/>
### Spec: `sharedAxis`
If you set `sharedAxis: false` for sub charts, then their axes are persisted in global space (shared). It's useful for when you creating multiple sub charts sharing their axes but have different parameters. For example,
- `basic-column`, `stacked-column`, `percent-column`
- `pie` and `donut`
<br/>
Here is an example.
```js
const spec = {
charts: {
'column': {
transform: { method: 'array', },
sharedAxis: true,
axis: { ... },
parameter: { ... },
},
'stacked': {
transform: { method: 'array', },
sharedAxis: true,
axis: { ... }
parameter: { ... },
},
```
<br/>
### Spec: `parameter`
| Field Name | Available Values (type) | Description |
| --- | --- | --- |
|`valueType` | `string` | Parameter which has string value |
|`valueType` | `int` | Parameter which has int value |
|`valueType` | `float` | Parameter which has float value |
|`valueType` | `boolean` | Parameter which has boolean value used with `checkbox` widget usually |
|`valueType` | `JSON` | Parameter which has JSON value used with `textarea` widget usually. `defaultValue` should be `""` (empty string). This ||`defaultValue` | (any) | Default value of this parameter. `JSON` type should have `""` (empty string) |
|`description` | (string) | Description of this parameter. This value will be parsed as HTML for pretty output |
|`widget` | `input` | Use [input](https://developer.mozilla.org/en/docs/Web/HTML/Element/input) widget. This is the default widget (if `widget` is undefined)|
|`widget` | `checkbox` | Use [checkbox](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/checkbox) widget. |
|`widget` | `textarea` | Use [textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) widget. |
|`widget` | `option` | Use [select + option](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select) widget. This parameter should have `optionValues` field as well. |
|`optionValues` | (Array<string>) | Available option values used with the `option` widget |
<br/>
Here is an example.
```js
parameter: {
// string type, input widget
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
// boolean type, checkbox widget
'inverted': { widget: 'checkbox', valueType: 'boolean', defaultValue: false, description: 'invert x and y axes', },
// string type, option widget with `optionValues`
'graphType': { widget: 'option', valueType: 'string', defaultValue: 'line', description: 'graph type', optionValues: [ 'line', 'smoothedLine', 'step', ], },
// HTML in `description`
'dateFormat': { valueType: 'string', defaultValue: '', description: 'format of date (<a href="https://docs.amcharts.com/3/javascriptcharts/AmGraph#dateFormat">doc</a>) (e.g YYYY-MM-DD)', },
// JSON type, textarea widget
'yAxisGuides': { widget: 'textarea', valueType: 'JSON', defaultValue: '', description: 'guides of yAxis ', },
```
<br/>
### Spec: `transform`
| Field Name | Available Values (type) | Description |
| --- | --- | --- |
|`method` | `object` | designed for rows requiring object manipulation |
|`method` | `array` | designed for rows requiring array manipulation |
|`method` | `array:2-key` | designed for xyz charts (e.g bubble chart) |
|`method` | `drill-down` | designed for drill-down charts |
|`method` | `raw` | will return the original `tableData.rows` |
<br/>
Whatever you specified as `transform.method`, the `transformer` value will be always function for lazy computation.
```js
// advanced-transformation.util#getTransformer
if (transformSpec.method === 'raw') {
transformer = () => { return rows; }
} else if (transformSpec.method === 'array') {
transformer = () => {
...
return { ... }
}
}
```
Here is actual usage.
```js
class AwesomeVisualization extends Visualization {
constructor(...) { /** setup your spec */ }
...
// `render` will be called whenever `axis` or `parameter` are changed
render(data) {
const { chart, parameter, column, transformer, } = data
if (chart === 'line') {
const transformed = transformer()
// draw line chart
} else if (chart === 'no-group') {
const transformed = transformer()
// draw no-group chart
}
}
...
}
```

88
docs/development/writingzeppelininterpreter.md → docs/development/writing_zeppelin_interpreter.md
View file

@ -29,7 +29,7 @@ Apache Zeppelin Interpreter is a language backend. For example to use scala code
Every Interpreters belongs to an **InterpreterGroup**.
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/interpreter.png" />
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="{{BASE_PATH}}/assets/themes/zeppelin/img/interpreter.png" />
[InterpreterSetting](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java) is configuration of a given [InterpreterGroup](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java) and a unit of start/stop interpreter.
All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via **[Thrift](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift)**.
@ -40,7 +40,49 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
## Make your own Interpreter
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
For your interpreter project, you need to make `interpreter-parent` as your parent project and use plugin `maven-enforcer-plugin`, `maven-dependency-plugin` and `maven-resources-plugin`. Here's one sample pom.xml
```xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
...
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
```
You should include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` as your interpreter's dependency in `pom.xml`. Bes
And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.
@ -61,18 +103,21 @@ Here is an example of `interpreter-setting.json` on your own interpreter.
"envName": null,
"propertyName": "property.1.name",
"defaultValue": "propertyDefaultValue",
"description": "Property description"
"description": "Property description",
"type": "textarea"
},
"properties2": {
"envName": PROPERTIES_2,
"propertyName": null,
"defaultValue": "property2DefaultValue",
"description": "Property 2 description"
"description": "Property 2 description",
"type": "textarea"
}, ...
},
"editor": {
"language": "your-syntax-highlight-language",
"editOnDblClick": false
"editOnDblClick": false,
"completionKey": "TAB"
}
},
{
@ -83,7 +128,7 @@ Here is an example of `interpreter-setting.json` on your own interpreter.
Finally, Zeppelin uses static initialization with the following:
```
```java
static {
Interpreter.register("MyInterpreterName", MyClassName.class.getName());
}
@ -112,7 +157,7 @@ If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to <code>[zeppelin-web/bower.json](https://github.com/apache/zeppelin/blob/master/zeppelin-web/bower.json)</code> (when built, <code>[zeppelin-web/src/index.html](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/index.html)</code> will be changed automatically).
2. Add `language` field to `editor` object. Note that if you don't specify language field, your interpreter will use plain text mode for syntax highlighting. Let's say you want to set your language to `java`, then add:
```
```json
"editor": {
"language": "java"
}
@ -121,11 +166,24 @@ If you want to add a new set of syntax highlighting,
### Edit on double click
If your interpreter uses mark-up language such as markdown or HTML, set `editOnDblClick` to `true` so that text editor opens on pargraph double click and closes on paragraph run. Otherwise set it to `false`.
```
```json
"editor": {
"editOnDblClick": false
}
```
### Completion key (Optional)
By default, `Ctrl+dot(.)` brings autocompletion list in the editor.
Through `completionKey`, each interpreter can configure autocompletion key.
Currently `TAB` is only available option.
```json
"editor": {
"completionKey": "TAB"
}
```
## Install your interpreter binary
Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies.
@ -143,7 +201,7 @@ To configure your interpreter you need to follow these steps:
Property value is comma separated [INTERPRETER\_CLASS\_NAME].
For example,
```
```xml
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter</value>
@ -167,7 +225,7 @@ Note that the first interpreter configuration in zeppelin.interpreters will be t
For example,
```
```scala
%myintp
val a = "My interpreter"
@ -177,11 +235,11 @@ println(a)
### 0.6.0 and later
Inside of a note, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter.
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. If you omit [INTERPRETER\_NAME], then first available interpreter will be selected in the [INTERPRETER\_GROUP].
Likewise, if you skip [INTERPRETER\_GROUP], then [INTERPRETER\_NAME] will be chosen from default interpreter group.
You can omit either `[INTERPRETER\_GROUP]` or `[INTERPRETER\_NAME]`. If you omit `[INTERPRETER\_NAME]`, then first available interpreter will be selected in the `[INTERPRETER\_GROUP]`.
Likewise, if you skip `[INTERPRETER\_GROUP]`, then `[INTERPRETER\_NAME]` will be chosen from default interpreter group.
For example, if you have two interpreter myintp1 and myintp2 in group mygrp, you can call myintp1 like
For example, if you have two interpreter `myintp1` and `myintp2` in group `mygrp`, you can call myintp1 like
```
%mygrp.myintp1
@ -189,7 +247,7 @@ For example, if you have two interpreter myintp1 and myintp2 in group mygrp, you
codes for myintp1
```
and you can call myintp2 like
and you can call `myintp2` like
```
%mygrp.myintp2
@ -197,7 +255,7 @@ and you can call myintp2 like
codes for myintp2
```
If you omit your interpreter name, it'll select first available interpreter in the group ( myintp1 ).
If you omit your interpreter name, it'll select first available interpreter in the group ( `myintp1` ).
```
%mygrp

63
docs/displaysystem/basicdisplaysystem.md
View file

@ -1,63 +0,0 @@
---
layout: page
title: "Basic Display System in Apache Zeppelin"
description: "There are 3 basic display systems in Apache Zeppelin. By default, Zeppelin prints interpreter responce as a plain text using text display system. With %html directive, Zeppelin treats your output as HTML. You can also simply use %table display system to leverage Zeppelin's built in visualization."
group: display
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Basic Display System in Apache Zeppelin
<div id="toc"></div>
## Text
By default, Apache Zeppelin prints interpreter response as a plain text using `text` display system.
<img src="../assets/themes/zeppelin/img/screenshots/display_text.png" />
You can explicitly say you're using `text` display system.
<img src="../assets/themes/zeppelin/img/screenshots/display_text1.png" />
## Html
With `%html` directive, Zeppelin treats your output as HTML
<img src="../assets/themes/zeppelin/img/screenshots/display_html.png" />
### Mathematical expressions
HTML display system automatically formats mathematical expression using [MathJax](https://www.mathjax.org/). You can use
`\\( INLINE EXPRESSION \\)` and `$$ EXPRESSION $$` to format. For example
<img src="../assets/themes/zeppelin/img/screenshots/display_formula.png" />
## Table
If you have data that row separated by `\n` (newline) and column separated by `\t` (tab) with first row as header row, for example
<img src="../assets/themes/zeppelin/img/screenshots/display_table.png" />
You can simply use `%table` display system to leverage Zeppelin's built in visualization.
<img src="../assets/themes/zeppelin/img/screenshots/display_table1.png" />
If table contents start with `%html`, it is interpreted as an HTML.
<img src="../assets/themes/zeppelin/img/screenshots/display_table_html.png" />
> **Note :** Display system is backend independent.

Some files were not shown because too many files have changed in this diff Show more