mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
mfelgamal
GitHub
commit
48d0a01741
220 changed files with 5490 additions and 2245 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -47,6 +47,7 @@ zeppelin-web/bower_components
|
|||
# R
|
||||
/r/lib/
|
||||
.Rhistory
|
||||
/R/
|
||||
|
||||
# project level
|
||||
/logs/
|
||||
|
|
@ -108,3 +109,6 @@ tramp
|
|||
|
||||
# Generated by zeppelin-examples
|
||||
/helium
|
||||
|
||||
# tmp files
|
||||
/tmp/
|
||||
|
|
|
|||
28
.travis.yml
28
.travis.yml
|
|
@ -21,6 +21,7 @@ cache:
|
|||
directories:
|
||||
- .spark-dist
|
||||
- ${HOME}/.m2/repository/.cache/maven-download-plugin
|
||||
- .node_modules
|
||||
|
||||
addons:
|
||||
apt:
|
||||
|
|
@ -33,44 +34,49 @@ addons:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
# Test all modules with spark-2.0.0-preview and scala 2.11
|
||||
# Test License compliance using RAT tool
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Dspark.version=2.0.0-preview -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
|
||||
|
||||
# Test all modules with spark 2.0.0 and scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
|
||||
|
||||
# Test all modules with scala 2.10
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
|
||||
|
||||
# Test all modules with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
|
||||
|
||||
# Test spark module for 1.5.2
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.4.1
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.3.1
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.2.2
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.1.1
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
|
||||
|
||||
# Test selenium with spark module for 1.6.1
|
||||
- jdk: "oraclejdk7"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
|
||||
before_install:
|
||||
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
|
||||
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
|
||||
- mkdir -p ~/R
|
||||
- echo 'R_LIBS=~/R' > ~/.Renviron
|
||||
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
|
||||
|
|
@ -89,6 +95,7 @@ before_script:
|
|||
|
||||
script:
|
||||
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
|
||||
- rm -rf .node_modules; cp -r zeppelin-web/node_modules .node_modules
|
||||
|
||||
after_success:
|
||||
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
|
||||
|
|
@ -104,3 +111,4 @@ after_failure:
|
|||
|
||||
after_script:
|
||||
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
|
||||
|
||||
|
|
|
|||
3
LICENSE
3
LICENSE
|
|
@ -242,7 +242,8 @@ The following components are provided under the MIT-style license. See project l
|
|||
The text of each license is also included at licenses/LICENSE-[project]-[version].txt.
|
||||
|
||||
(MIT Style) jekyll-table-of-contents (https://github.com/ghiculescu/jekyll-table-of-contents) - https://github.com/ghiculescu/jekyll-table-of-contents/blob/master/LICENSE.txt
|
||||
|
||||
(MIT Style) lunr.js (https://github.com/olivernn/lunr.js) - https://github.com/olivernn/lunr.js/blob/v0.7.1/LICENSE
|
||||
|
||||
========================================================================
|
||||
Apache licenses
|
||||
========================================================================
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -1,4 +1,4 @@
|
|||
#Zeppelin
|
||||
# Apache Zeppelin
|
||||
|
||||
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
|
||||
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
|
||||
|
|
@ -93,9 +93,9 @@ _Notes:_
|
|||
|
||||
#### Install maven
|
||||
```
|
||||
wget http://www.eu.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
|
||||
sudo tar -zxf apache-maven-3.3.3-bin.tar.gz -C /usr/local/
|
||||
sudo ln -s /usr/local/apache-maven-3.3.3/bin/mvn /usr/local/bin/mvn
|
||||
wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
|
||||
sudo tar -zxf apache-maven-3.3.9-bin.tar.gz -C /usr/local/
|
||||
sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/local/bin/mvn
|
||||
```
|
||||
|
||||
_Notes:_
|
||||
|
|
@ -217,6 +217,7 @@ Here're some examples:
|
|||
|
||||
```sh
|
||||
# build with spark-2.0, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
|
||||
|
||||
# build with spark-1.6, scala-2.10
|
||||
|
|
@ -306,6 +307,7 @@ For configuration details check __`./conf`__ subdirectory.
|
|||
To produce a Zeppelin package compiled with Scala 2.11, use the -Pscala-2.11 profile:
|
||||
|
||||
```
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Pscala-2.11 -DskipTests clean install
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -81,13 +81,6 @@ if not defined JAVA_OPTS (
|
|||
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
|
||||
)
|
||||
|
||||
if not defined ZEPPELIN_INTP_JAVA_OPTS (
|
||||
set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
|
||||
)
|
||||
|
||||
if not defined ZEPPELIN_INTP_MEM (
|
||||
set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM%
|
||||
)
|
||||
|
||||
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%
|
||||
|
||||
|
|
|
|||
|
|
@ -121,15 +121,6 @@ JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPEL
|
|||
JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
|
||||
export JAVA_OPTS
|
||||
|
||||
# jvm options for interpreter process
|
||||
if [[ -z "${ZEPPELIN_INTP_JAVA_OPTS}" ]]; then
|
||||
export ZEPPELIN_INTP_JAVA_OPTS="${ZEPPELIN_JAVA_OPTS}"
|
||||
fi
|
||||
|
||||
if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
|
||||
export ZEPPELIN_INTP_MEM="${ZEPPELIN_MEM}"
|
||||
fi
|
||||
|
||||
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
|
||||
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
|
||||
export JAVA_INTP_OPTS
|
||||
|
|
|
|||
|
|
@ -46,6 +46,14 @@ if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" (
|
|||
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"!ZEPPELIN_INTERPRETER_JAR!"
|
||||
)
|
||||
|
||||
REM add test classes for unittest
|
||||
if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\test-classes" (
|
||||
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\test-classes"
|
||||
)
|
||||
if exist "%ZEPPELIN_HOME%\zeppelin-zengine\target\test-classes" (
|
||||
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-zengine\target\test-classes"
|
||||
)
|
||||
|
||||
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib"
|
||||
call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%"
|
||||
|
||||
|
|
|
|||
|
|
@ -63,6 +63,15 @@ else
|
|||
ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}"
|
||||
fi
|
||||
|
||||
# add test classes for unittest
|
||||
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/test-classes" ]]; then
|
||||
ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/test-classes"
|
||||
fi
|
||||
if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/test-classes" ]]; then
|
||||
ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/test-classes"
|
||||
fi
|
||||
|
||||
|
||||
addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
|
||||
addJarInDirForIntp "${INTERPRETER_DIR}"
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ user3 = password4, role2
|
|||
### A sample for configuring Active Directory Realm
|
||||
#activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
|
||||
#activeDirectoryRealm.systemUsername = userNameA
|
||||
|
||||
#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html
|
||||
#activeDirectoryRealm.systemPassword = passwordA
|
||||
#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks
|
||||
#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
||||
#activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
|
||||
|
|
@ -42,6 +45,11 @@ user3 = password4, role2
|
|||
#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
|
||||
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
|
||||
|
||||
### A sample for configuring ZeppelinHub Realm
|
||||
#zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
|
||||
## Url of ZeppelinHub
|
||||
#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
|
||||
#securityManager.realms = $zeppelinHubRealm
|
||||
|
||||
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ REM set JAVA_HOME=
|
|||
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
|
||||
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
|
||||
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
|
||||
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
|
||||
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
|
||||
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options.
|
||||
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options.
|
||||
|
||||
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
|
||||
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.
|
||||
|
|
@ -35,6 +35,7 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep
|
|||
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
|
||||
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
|
||||
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
|
||||
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?
|
||||
|
||||
|
||||
REM Spark interpreter configuration
|
||||
|
|
@ -62,7 +63,7 @@ REM
|
|||
REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default.
|
||||
REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default.
|
||||
REM set ZEPPELIN_SPARK_IMPORTIMPLICIT REM Import implicits, UDF collection, and sql if set true. true by default.
|
||||
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default.
|
||||
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of Spark SQL result to display. 1000 by default.
|
||||
|
||||
REM ZeppelinHub connection configuration
|
||||
REM
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@
|
|||
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
|
||||
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
|
||||
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
|
||||
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
|
||||
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
|
||||
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options.
|
||||
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options.
|
||||
|
||||
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
|
||||
# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default.
|
||||
|
|
@ -36,6 +36,7 @@
|
|||
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
|
||||
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
|
||||
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
|
||||
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
|
||||
|
||||
#### Spark interpreter configuration ####
|
||||
|
||||
|
|
@ -62,7 +63,7 @@
|
|||
# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default.
|
||||
# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default.
|
||||
# export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default.
|
||||
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
|
||||
# export ZEPPELIN_SPARK_MAXRESULT # Max number of Spark SQL result to display. 1000 by default.
|
||||
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -164,6 +164,12 @@
|
|||
<description>notebook persistence layer implementation</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>zeppelin.notebook.one.way.sync</name>
|
||||
<value>false</value>
|
||||
<description>If there are multiple notebook storages, should we treat the first one as the only source of truth?</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.dir</name>
|
||||
<value>interpreter</value>
|
||||
|
|
@ -184,7 +190,7 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.group.order</name>
|
||||
<value>"spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase</value>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery</value>
|
||||
<description></description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ function make_binary_release() {
|
|||
|
||||
cp -r "${WORKING_DIR}/zeppelin" "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
|
||||
cd "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
|
||||
./dev/change_scala_version.sh 2.11
|
||||
echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}"
|
||||
mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}
|
||||
if [[ $? -ne 0 ]]; then
|
||||
|
|
@ -102,8 +103,8 @@ function make_binary_release() {
|
|||
|
||||
git_clone
|
||||
make_source_package
|
||||
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell"
|
||||
make_binary_release all "-Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pscala-2.11"
|
||||
make_binary_release netinst "-Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pscala-2.11 -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell,!bigquery"
|
||||
|
||||
# remove non release files and dirs
|
||||
rm -rf "${WORKING_DIR}/zeppelin"
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ NC='\033[0m' # No Color
|
|||
RELEASE_VERSION="$1"
|
||||
GIT_TAG="$2"
|
||||
|
||||
PUBLISH_PROFILES="-Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
PUBLISH_PROFILES="-Pbuild-distr -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
PROJECT_OPTIONS="-pl !zeppelin-distribution"
|
||||
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
|
||||
NEXUS_PROFILE="153446d1ac37c4"
|
||||
|
|
@ -93,10 +93,10 @@ function publish_to_maven() {
|
|||
tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)"
|
||||
|
||||
# build with scala-2.10
|
||||
echo "mvn clean install -Ppublish-distr \
|
||||
echo "mvn clean install -DskipTests \
|
||||
-Dmaven.repo.local=${tmp_repo} -Pscala-2.10 \
|
||||
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}"
|
||||
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" -Pscala-2.10 \
|
||||
mvn clean install -DskipTests -Dmaven.repo.local="${tmp_repo}" -Pscala-2.10 \
|
||||
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Build with scala 2.10 failed."
|
||||
|
|
@ -106,10 +106,10 @@ function publish_to_maven() {
|
|||
# build with scala-2.11
|
||||
"${BASEDIR}/change_scala_version.sh" 2.11
|
||||
|
||||
echo "mvn clean install -Ppublish-distr \
|
||||
echo "mvn clean install -DskipTests \
|
||||
-Dmaven.repo.local=${tmp_repo} -Pscala-2.11 \
|
||||
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}"
|
||||
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" -Pscala-2.11 \
|
||||
mvn clean install -DskipTests -Dmaven.repo.local="${tmp_repo}" -Pscala-2.11 \
|
||||
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Build with scala 2.11 failed."
|
||||
|
|
|
|||
94
docs/CONTRIBUTING.md
Normal file
94
docs/CONTRIBUTING.md
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
# Contributing to Apache Zeppelin Documentation
|
||||
|
||||
## Folder Structure
|
||||
`docs/` folder is organized as below:
|
||||
|
||||
```
|
||||
docs/
|
||||
├── _includes/themes/zeppelin
|
||||
│ ├── _navigation.html
|
||||
│ └── default.html
|
||||
├── _layouts
|
||||
├── _plugins
|
||||
├── assets/themes/zeppelin -> {ASSET_PATH}
|
||||
│ ├── bootstrap
|
||||
│ ├── css
|
||||
│ ├── img
|
||||
│ └── js
|
||||
├── development/ *.md
|
||||
├── displaysystem/ *.md
|
||||
├── install/ *.md
|
||||
├── interpreter/ *.md
|
||||
├── manual/ *.md
|
||||
├── quickstart/ *.md
|
||||
├── rest-api/ *.md
|
||||
├── security/ *.md
|
||||
├── storage/ *.md
|
||||
├── Gemfile
|
||||
├── Gemfile.lock
|
||||
├── _config.yml
|
||||
├── index.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
- `_navigation.html`: the dropdown menu in navbar
|
||||
- `default.html` & `_layouts/`: define default HTML layout
|
||||
- `_plugins/`: custom plugin `*.rb` files can be placed in this folder. See [jekyll/plugins](https://jekyllrb.com/docs/plugins/) for the further information.
|
||||
- `{ASSET_PATH}/css/style.css`: extra css components can be defined
|
||||
- `{ASSET_PATH}/img/docs-img/`: image files used for document pages can be placed in this folder
|
||||
- `{ASSET_PATH}/js/`: extra `.js` files can be placed
|
||||
- `Gemfile`: defines bundle dependencies. They will be installed by `bundle install`.
|
||||
- `Gemfile.lock`: when you run `bundle install`, bundler will persist all gems name and their version to this file. For the more details, see [Bundle "The Gemfile Lock"](http://bundler.io/v1.10/man/bundle-install.1.html#THE-GEMFILE-LOCK)
|
||||
- `documentation_group`: `development/`, `displaysystem/`, `install/`, `interpreter/`...
|
||||
- `_config.yml`: defines configuration options for docs website. See [jekyll/configuration](https://jekyllrb.com/docs/configuration/) for the other available config variables.
|
||||
- `index.md`: the main page of `http://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>/`
|
||||
|
||||
|
||||
## Markdown
|
||||
Zeppelin documentation pages are written with [Markdown](http://daringfireball.net/projects/markdown/). It is possible to use [GitHub flavored syntax](https://help.github.com/categories/writing-on-github/) and intermix plain HTML.
|
||||
|
||||
## Front matter
|
||||
Every page contains [YAML front matter](https://jekyllrb.com/docs/frontmatter/) block in their header. Don't forget to wrap the front matter list with triple-dashed lines(`---`) like below.
|
||||
The document page should start this triple-dashed lines. Or you will face 404 error, since Jekyll can't find the page.
|
||||
|
||||
```
|
||||
---
|
||||
layout: page
|
||||
title: "Apache Zeppelin Tutorial"
|
||||
description: "This tutorial page contains a short walk-through tutorial that uses Apache Spark backend. Please note that this tutorial is valid for Spark 1.3 and higher."
|
||||
group: quickstart
|
||||
---
|
||||
```
|
||||
|
||||
- `layout`: the default layout is `page` which is defined in `_layout/page.html`.
|
||||
- `title`: the title for the document. Please note that if it needs to include `Zeppelin`, it should be `Apache Zeppelin`, not `Zeppelin`.
|
||||
- `description`: a short description for the document. One or two sentences would be enough. This description also will be shown as an extract sentence when people search pages.
|
||||
- `group`: a category of the document page
|
||||
|
||||
## Headings
|
||||
All documents are structured with headings. From these headings, you can automatically generate a **Table of Contents**. There is a simple rule for Zeppelin docs headings.
|
||||
|
||||
```
|
||||
# Level-1 heading <- used only for the main title
|
||||
## Level-2 heading <- start with this
|
||||
### Level-3 heading
|
||||
#### Level-4 heading <- won't be converted in TOC from this level
|
||||
```
|
||||
|
||||
## Table of contents(TOC)
|
||||
|
||||
```
|
||||
<div id="toc"></div>
|
||||
```
|
||||
|
||||
Add this line below `# main title` in order to generate a **Table of Contents**. Headings until `### (Level-3 heading)` are included to TOC.
|
||||
|
||||
|
||||
Default setting options for TOC are definded in [here](https://github.com/apache/zeppelin/blob/master/docs/assets/themes/zeppelin/js/toc.js#L4).
|
||||
|
||||
|
||||
## Adding new pages
|
||||
If you're going to create new pages, there are some spots you need to add the location of the page.
|
||||
|
||||
- **Dropdown menu in navbar**: add your docs location to [_navigation.html](https://github.com/apache/zeppelin/blob/master/docs/_includes/themes/zeppelin/_navigation.html)
|
||||
- **Main index**: add your docs below [What is the next?](http://zeppelin.apache.org/docs/latest/#what-is-the-next) section in [index.md](https://github.com/apache/zeppelin/blob/master/docs/index.md) with a short description. No need to do this if the page is for **Interpreters**.
|
||||
|
|
@ -1,41 +1,55 @@
|
|||
## Apache Zeppelin documentation
|
||||
|
||||
This readme will walk you through building the Zeppelin documentation, which is included here with the Zeppelin source code.
|
||||
# Apache Zeppelin documentation
|
||||
|
||||
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest) is also generated from the files found in here.
|
||||
|
||||
## Build documentation
|
||||
See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
|
||||
Zeppelin is using [Jekyll](https://jekyllrb.com/) which is a static site generator and [Github Pages](https://pages.github.com/) as a site publisher. For the more details, see [help.github.com/articles/about-github-pages-and-jekyll/](https://help.github.com/articles/about-github-pages-and-jekyll/).
|
||||
|
||||
**Requirements**
|
||||
|
||||
```
|
||||
ruby --version >= 2.0.0
|
||||
gem install bundler
|
||||
# go to /docs under your Zeppelin source
|
||||
bundle install
|
||||
# ruby --version >= 2.0.0
|
||||
# Install Bundler using gem
|
||||
gem install bundler
|
||||
|
||||
cd $ZEPPELIN_HOME/docs
|
||||
# Install all dependencies declared in the Gemfile
|
||||
bundle install
|
||||
```
|
||||
|
||||
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
|
||||
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
|
||||
|
||||
*On OS X 10.9 you may need to do "xcode-select --install"*
|
||||
On OS X 10.9, you may need to do
|
||||
|
||||
```
|
||||
xcode-select --install
|
||||
```
|
||||
|
||||
## Run website locally
|
||||
If you don't want to encounter uglily rendered pages, run the documentation site in your local first.
|
||||
|
||||
In `$ZEPPELIN_HOME/docs`,
|
||||
|
||||
```
|
||||
bundle exec jekyll serve --watch
|
||||
```
|
||||
|
||||
Using the above command, Jekyll will start a web server at `http://localhost:4000` and watch the `/docs` directory to update.
|
||||
|
||||
|
||||
## Run website
|
||||
|
||||
bundle exec jekyll serve --watch
|
||||
## Contribute to Zeppelin documentation
|
||||
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://github.com/apache/zeppelin/blob/master/docs/CONTRIBUTING.md).
|
||||
|
||||
|
||||
## Adding a new page
|
||||
|
||||
rake page name="new-page.md"
|
||||
|
||||
|
||||
## Bumping up version in a new release
|
||||
## For committers only
|
||||
### Bumping up version in a new release
|
||||
|
||||
* `ZEPPELIN_VERSION` and `BASE_PATH` property in _config.yml
|
||||
|
||||
## Deploy to ASF svnpubsub infra (for committers only)
|
||||
### Deploy to ASF svnpubsub infra
|
||||
1. generate static website in `./_site`
|
||||
|
||||
```
|
||||
# go to /docs under Zeppelin source
|
||||
bundle exec jekyll build --safe
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@
|
|||
<li><a href="{{BASE_PATH}}/manual/notebookashomepage.html">Customize Zeppelin Homepage</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>More</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/upgrade.html">Upgrade Zeppelin Version</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
|
|
@ -103,6 +102,12 @@
|
|||
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
|
||||
<li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Advanced</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-yarn-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-on-mesos-mode">Zeppelin on Spark Cluster Mode (Mesos)</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Contibute</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li>
|
||||
|
|
@ -111,7 +116,14 @@
|
|||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<ul class="nav navbar-nav">
|
||||
<li>
|
||||
<a href="{{BASE_PATH}}/search.html" class="nav-search-link">
|
||||
<span class="fa fa-search nav-search-icon"></span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav><!--/.navbar-collapse -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>{{ page.title }}</title>
|
||||
<title>Apache Zeppelin {{ site.ZEPPELIN_VERSION }} Documentation: {{ page.title }}</title>
|
||||
{% if page.description %}<meta name="description" content="{{ page.description }}">{% endif %}
|
||||
<meta name="author" content="{{ site.author.name }}">
|
||||
|
||||
|
|
@ -34,6 +34,8 @@
|
|||
<script src="{{ ASSET_PATH }}/js/docs.js"></script>
|
||||
<script src="{{ ASSET_PATH }}/js/anchor.min.js"></script>
|
||||
<script src="{{ ASSET_PATH }}/js/toc.js"></script>
|
||||
<script src="{{ ASSET_PATH }}/js/lunr.min.js"></script>
|
||||
<script src="{{ ASSET_PATH }}/js/search.js"></script>
|
||||
|
||||
<!-- atom & rss feed -->
|
||||
<link href="{{ BASE_PATH }}{{ site.JB.atom_path }}" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed">
|
||||
|
|
|
|||
|
|
@ -438,11 +438,10 @@ a.anchor {
|
|||
.content table {
|
||||
display: block;
|
||||
width: 100%;
|
||||
overflow: auto;
|
||||
word-break: normal;
|
||||
word-break: keep-all;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
font-size: 90%;
|
||||
font-size: 87%;
|
||||
margin-top: 16px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
|
@ -569,6 +568,30 @@ a.anchorjs-link:hover { text-decoration: none; }
|
|||
margin-left: -18px;
|
||||
}
|
||||
|
||||
/* Search Page */
|
||||
#search p {
|
||||
font-size: 30px;
|
||||
font-weight: bold;
|
||||
color: black;
|
||||
}
|
||||
|
||||
#search_results p {
|
||||
font-size: 13px;
|
||||
font-weight: 400;
|
||||
}
|
||||
|
||||
#search_results a {
|
||||
vertical-align: super;
|
||||
font-size: 16px;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
#search_results .link {
|
||||
font-size: 13px;
|
||||
color: #008000;
|
||||
padding-bottom: 3px;
|
||||
}
|
||||
|
||||
/* Custom, iPhone Retina */
|
||||
@media only screen and (max-width: 480px) {
|
||||
.jumbotron h1 {
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 245 KiB After Width: | Height: | Size: 195 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 76 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/spark_ui.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/spark_ui.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 201 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/standalone_conf.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/standalone_conf.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 180 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 95 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 212 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
BIN
docs/assets/themes/zeppelin/img/spark_logo.png
Normal file
BIN
docs/assets/themes/zeppelin/img/spark_logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
7
docs/assets/themes/zeppelin/js/lunr.min.js
vendored
Normal file
7
docs/assets/themes/zeppelin/js/lunr.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
73
docs/assets/themes/zeppelin/js/search.js
Normal file
73
docs/assets/themes/zeppelin/js/search.js
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
---
|
||||
|
||||
---
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
jQuery(function() {
|
||||
window.idx = lunr(function () {
|
||||
this.field('id');
|
||||
this.field('title');
|
||||
this.field('content', { boost: 10 });
|
||||
this.field('group');
|
||||
});
|
||||
|
||||
window.data = $.getJSON('search_data.json');
|
||||
window.data.then(function(loaded_data){
|
||||
$.each(loaded_data, function(index, value){
|
||||
window.idx.add(
|
||||
$.extend({ "id": index }, value)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
$("#site_search").keyup(function(event){
|
||||
event.preventDefault();
|
||||
var query = $("#search_box").val();
|
||||
var results = window.idx.search(query);
|
||||
display_search_results(results);
|
||||
});
|
||||
|
||||
$('html').bind('keypress', function(event){
|
||||
// Since keyup() is operated at the above, disable 'Enter Key' press.
|
||||
if(event.keyCode == 13) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
function display_search_results(results) {
|
||||
var $search_results = $("#search_results");
|
||||
var zeppelin_version = {{site.ZEPPELIN_VERSION | jsonify}};
|
||||
var base_url = {{site.JB.BASE_PATH | jsonify}};
|
||||
var prod_url = {{site.production_url | jsonify}};
|
||||
|
||||
window.data.then(function(loaded_data) {
|
||||
if (results.length) {
|
||||
$search_results.empty();
|
||||
$search_results.prepend('<p class="">Found '+results.length+' result(s)</p><hr>');
|
||||
|
||||
results.forEach(function(result) {
|
||||
var item = loaded_data[result.ref];
|
||||
var appendString = '<a href="'+base_url+item.url.trim()+'">'+item.title+'</a><div class="link">'+prod_url+base_url+item.url.trim()+'</div><p>'+item.excerpt+'</p><br/>';
|
||||
|
||||
$search_results.append(appendString);
|
||||
});
|
||||
} else {
|
||||
$search_results.html('<p>Your search did not match any documents.<br/>Make sure that all words are spelled correctly or try more general keywords.</p>');
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
@ -1,9 +1,23 @@
|
|||
---
|
||||
layout: page
|
||||
title: "How to contribute"
|
||||
description: "How to contribute"
|
||||
title: "Contributing to Apache Zeppelin (Code)"
|
||||
description: "How can you contribute to Apache Zeppelin project? This document covers from setting up your develop environment to making a pull request on Github."
|
||||
group: development
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Contributing to Apache Zeppelin ( Code )
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,23 @@
|
|||
---
|
||||
layout: page
|
||||
title: "How to contribute (website)"
|
||||
description: "How to contribute (website)"
|
||||
title: "Contributing to Apache Zeppelin (Website)"
|
||||
description: "How can you contribute to Apache Zeppelin project website? This document covers from building Zeppelin documentation site to making a pull request on Github."
|
||||
group: development
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Contributing to Apache Zeppelin ( Website )
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Writing Zeppelin Application"
|
||||
description: ""
|
||||
title: "Writing a new Application(Experimental)"
|
||||
description: "Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. Make your own Application in Apache Zeppelin is quite easy."
|
||||
group: development
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Writing Zeppelin Interpreter"
|
||||
description: ""
|
||||
title: "Writing a New Interpreter"
|
||||
description: "Apache Zeppelin Interpreter is a language backend. Every Interpreters belongs to an InterpreterGroup. Interpreters in the same InterpreterGroup can reference each other."
|
||||
group: development
|
||||
---
|
||||
<!--
|
||||
|
|
@ -156,7 +156,6 @@ println(a)
|
|||
|
||||
### 0.6.0 and later
|
||||
Inside of a notebook, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter.
|
||||
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
|
||||
|
||||
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. If you omit [INTERPRETER\_NAME], then first available interpreter will be selected in the [INTERPRETER\_GROUP].
|
||||
Likewise, if you skip [INTERPRETER\_GROUP], then [INTERPRETER\_NAME] will be chosen from default interpreter group.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Angular (backend API)"
|
||||
description: "Angular (backend API)"
|
||||
title: "Back-end Angular API in Apache Zeppelin"
|
||||
description: "Apache Zeppelin provides a gateway between your interpreter and your compiled AngularJS view templates. You can not only update scope variables from your interpreter but also watch them in the interpreter, which is JVM process."
|
||||
group: display
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Basic Display System"
|
||||
description: ""
|
||||
title: "Basic Display System in Apache Zeppelin"
|
||||
description: "There are 3 basic display systems in Apache Zeppelin. By default, Zeppelin prints interpreter responce as a plain text using text display system. With %html directive, Zeppelin treats your output as HTML. You can also simply use %table display system to leverage Zeppelin's built in visualization."
|
||||
group: display
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Angular (frontend API)"
|
||||
description: "Angular (frontend API)"
|
||||
title: "Front-end Angular API in Apache Zeppelin"
|
||||
description: "In addition to the back-end API to handle Angular objects binding, Apache Zeppelin exposes a simple AngularJS z object on the front-end side to expose the same capabilities."
|
||||
group: display
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
---
|
||||
layout: page
|
||||
title: Overview
|
||||
tagline: Less Development, More analysis!
|
||||
group: nav-right
|
||||
title:
|
||||
description:
|
||||
group:
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -49,7 +49,7 @@ Adding new language-backend is really simple. Learn [how to create your own inte
|
|||
#### Apache Spark integration
|
||||
Especially, Apache Zeppelin provides built-in [Apache Spark](http://spark.apache.org/) integration. You don't need to build a separate module, plugin or library for it.
|
||||
|
||||
<img class="img-responsive" src="./assets/themes/zeppelin/img/spark_logo.jpg" width="140px" />
|
||||
<img class="img-responsive" src="./assets/themes/zeppelin/img/spark_logo.png" width="140px" />
|
||||
|
||||
Apache Zeppelin with Spark integration provides
|
||||
|
||||
|
|
@ -62,7 +62,7 @@ For the further information about Apache Spark in Apache Zeppelin, please see [S
|
|||
<br />
|
||||
## Data visualization
|
||||
|
||||
Some basic charts are already included in Apache Zeppelin. Visualizations are not limited to SparkSQL query, any output from any language backend can be recognized and visualized.
|
||||
Some basic charts are already included in Apache Zeppelin. Visualizations are not limited to Spark SQL query, any output from any language backend can be recognized and visualized.
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
|
|
@ -133,7 +133,6 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
|
|||
* [Publish your Paragraph](./manual/publish.html) results into your external website
|
||||
* [Customize Zeppelin Homepage](./manual/notebookashomepage.html) with one of your notebooks
|
||||
* More
|
||||
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html): a guide for installing Apache Zeppelin on Vagrant virtual machine
|
||||
* [Upgrade Apache Zeppelin Version](./install/upgrade.html): a manual procedure of upgrading Apache Zeppelin version
|
||||
|
||||
####Interpreter
|
||||
|
|
@ -168,6 +167,11 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
|
|||
* [Shiro Authentication](./security/shiroauthentication.html)
|
||||
* [Notebook Authorization](./security/notebook_authorization.html)
|
||||
* [Data Source Authorization](./security/datasource_authorization.html)
|
||||
* Advanced
|
||||
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
|
||||
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
|
||||
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-on-yarn-mode)
|
||||
* [Zeppelin on Spark Cluster Mode (Mesos via Docker)](./install/spark_cluster_mode.html#spark-on-mesos-mode)
|
||||
* Contribute
|
||||
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
|
||||
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Getting Started"
|
||||
description: ""
|
||||
title: "Quick Start"
|
||||
description: "This page will help you to get started and guide you through installation of Apache Zeppelin, running it in the command line and basic configuration options."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
|
|
@ -94,6 +94,7 @@ Here are some examples with several options
|
|||
|
||||
```
|
||||
# build with spark-2.0, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
|
||||
|
||||
# build with spark-1.6, scala-2.10
|
||||
|
|
@ -194,10 +195,10 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>zepplin-env.sh</th>
|
||||
<th>zepplin-site.xml</th>
|
||||
<th>zeppelin-env.sh</th>
|
||||
<th>zeppelin-site.xml</th>
|
||||
<th>Default value</th>
|
||||
<th>Description</th>
|
||||
<th class="col-md-4">Description</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ZEPPELIN_PORT</td>
|
||||
|
|
@ -353,7 +354,7 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
|
||||
<td>zeppelin.notebook.azure.connectionString</td>
|
||||
<td></td>
|
||||
<td>The Azure storage account connection string<br />i.e. <code>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></code></td>
|
||||
<td>The Azure storage account connection string<br />i.e. <br/><code>DefaultEndpointsProtocol=https;<br/>AccountName=<accountName>;<br/>AccountKey=<accountKey></code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
|
||||
|
|
@ -373,6 +374,12 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
<td>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</td>
|
||||
<td>Comma separated list of notebook storage</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC</td>
|
||||
<td>zeppelin.notebook.one.way.sync</td>
|
||||
<td>false</td>
|
||||
<td>If there are multiple notebook storages, should we treat the first one as the only source of truth?</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ZEPPELIN_INTERPRETERS</td>
|
||||
<td>zeppelin.interpreters</td>
|
||||
|
|
@ -380,7 +387,10 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />
|
||||
...
|
||||
</td>
|
||||
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in notebook paragraph. </td>
|
||||
<td>
|
||||
Comma separated interpreter configurations [Class] <br/>
|
||||
<span style="font-style:italic">NOTE: This property is deprecated since Zeppelin-0.6.0 and will not be supported from Zeppelin-0.7.0</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ZEPPELIN_INTERPRETER_DIR</td>
|
||||
|
|
@ -394,4 +404,4 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
<td>1024000</td>
|
||||
<td>Size in characters of the maximum text message to be received by websocket.</td>
|
||||
</tr>
|
||||
</table>
|
||||
</table>
|
||||
|
|
|
|||
204
docs/install/spark_cluster_mode.md
Normal file
204
docs/install/spark_cluster_mode.md
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Apache Zeppelin on Spark cluster mode"
|
||||
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager(Standalone, Hadoop Yarn, Apache Mesos) with Apache Zeppelin using docker scripts."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Apache Zeppelin on Spark Cluster Mode
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
## Overview
|
||||
[Apache Spark](http://spark.apache.org/) has supported three cluster manager types([Standalone](http://spark.apache.org/docs/latest/spark-standalone.html), [Apache Mesos](http://spark.apache.org/docs/latest/running-on-mesos.html) and [Hadoop YARN](http://spark.apache.org/docs/latest/running-on-yarn.html)) so far.
|
||||
This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using [Docker](https://www.docker.com/) scripts.
|
||||
So [install docker](https://docs.docker.com/engine/installation/) on the machine first.
|
||||
|
||||
## Spark standalone mode
|
||||
[Spark standalone](http://spark.apache.org/docs/latest/spark-standalone.html) is a simple cluster manager included with Spark that makes it easy to set up a cluster.
|
||||
You can simply set up Spark standalone environment with below steps.
|
||||
|
||||
> **Note :** Since Apache Zeppelin and Spark use same `8080` port for their web UI, you might need to change `zeppelin.server.port` in `conf/zeppelin-site.xml`.
|
||||
|
||||
### 1. Build Docker file
|
||||
You can find docker script files under `scripts/docker/spark-cluster-managers`.
|
||||
|
||||
```
|
||||
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_standalone
|
||||
docker build -t "spark_standalone" .
|
||||
```
|
||||
|
||||
### 2. Run docker
|
||||
|
||||
```
|
||||
docker run -it \
|
||||
-p 8080:8080 \
|
||||
-p 7077:7077 \
|
||||
-p 8888:8888 \
|
||||
-p 8081:8081 \
|
||||
-h sparkmaster \
|
||||
--name spark_standalone \
|
||||
spark_standalone bash;
|
||||
```
|
||||
|
||||
### 3. Configure Spark interpreter in Zeppelin
|
||||
Set Spark master as `spark://<hostname>:7077` in Zeppelin **Interpreters** setting page.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/standalone_conf.png" />
|
||||
|
||||
### 4. Run Zeppelin with Spark interpreter
|
||||
After running single paragraph with Spark interpreter in Zeppelin, browse `https://<hostname>:8080` and check whether Spark cluster is running well or not.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/spark_ui.png" />
|
||||
|
||||
You can also simply verify that Spark is running well in Docker with below command.
|
||||
|
||||
```
|
||||
ps -ef | grep spark
|
||||
```
|
||||
|
||||
|
||||
## Spark on YARN mode
|
||||
You can simply set up [Spark on YARN](http://spark.apache.org/docs/latest/running-on-yarn.html) docker environment with below steps.
|
||||
|
||||
> **Note :** Since Apache Zeppelin and Spark use same `8080` port for their web UI, you might need to change `zeppelin.server.port` in `conf/zeppelin-site.xml`.
|
||||
|
||||
### 1. Build Docker file
|
||||
You can find docker script files under `scripts/docker/spark-cluster-managers`.
|
||||
|
||||
```
|
||||
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_yarn
|
||||
docker build -t "spark_yarn" .
|
||||
```
|
||||
|
||||
### 2. Run docker
|
||||
|
||||
```
|
||||
docker run -it \
|
||||
-p 5000:5000 \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-p 8088:8088 \
|
||||
-p 8042:8042 \
|
||||
-p 8030:8030 \
|
||||
-p 8031:8031 \
|
||||
-p 8032:8032 \
|
||||
-p 8033:8033 \
|
||||
-p 8080:8080 \
|
||||
-p 7077:7077 \
|
||||
-p 8888:8888 \
|
||||
-p 8081:8081 \
|
||||
-p 50010:50010 \
|
||||
-p 50075:50075 \
|
||||
-p 50020:50020 \
|
||||
-p 50070:50070 \
|
||||
--name spark_yarn \
|
||||
-h sparkmaster \
|
||||
spark_yarn bash;
|
||||
```
|
||||
|
||||
### 3. Verify running Spark on YARN.
|
||||
|
||||
You can simply verify the processes of Spark and YARN are running well in Docker with below command.
|
||||
|
||||
```
|
||||
ps -ef
|
||||
```
|
||||
|
||||
You can also check each application web UI for HDFS on `http://<hostname>:50070/`, YARN on `http://<hostname>:8088/cluster` and Spark on `http://<hostname>:8080/`.
|
||||
|
||||
### 4. Configure Spark interpreter in Zeppelin
|
||||
Set following configurations to `conf/zeppelin-env.sh`.
|
||||
|
||||
```
|
||||
export MASTER=yarn-client
|
||||
export HADOOP_CONF_DIR=[your_hadoop_conf_path]
|
||||
export SPARK_HOME=[your_spark_home_path]
|
||||
```
|
||||
|
||||
`HADOOP_CONF_DIR`(Hadoop configuration path) is defined in `/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf`.
|
||||
|
||||
Don't forget to set Spark `master` as `yarn-client` in Zeppelin **Interpreters** setting page like below.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png" />
|
||||
|
||||
### 5. Run Zeppelin with Spark interpreter
|
||||
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/yarn_applications.png" />
|
||||
|
||||
|
||||
|
||||
## Spark on Mesos mode
|
||||
You can simply set up [Spark on Mesos](http://spark.apache.org/docs/latest/running-on-mesos.html) docker environment with below steps.
|
||||
|
||||
|
||||
### 1. Build Docker file
|
||||
|
||||
```
|
||||
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_mesos
|
||||
docker build -t "spark_mesos" .
|
||||
```
|
||||
|
||||
|
||||
### 2. Run docker
|
||||
|
||||
```
|
||||
docker run --net=host -it \
|
||||
-p 8080:8080 \
|
||||
-p 7077:7077 \
|
||||
-p 8888:8888 \
|
||||
-p 8081:8081 \
|
||||
-p 8082:8082 \
|
||||
-p 5050:5050 \
|
||||
-p 5051:5051 \
|
||||
-p 4040:4040 \
|
||||
-h sparkmaster \
|
||||
--name spark_mesos \
|
||||
spark_mesos bash;
|
||||
```
|
||||
|
||||
### 3. Verify running Spark on Mesos.
|
||||
|
||||
You can simply verify the processes of Spark and Mesos are running well in Docker with below command.
|
||||
|
||||
```
|
||||
ps -ef
|
||||
```
|
||||
|
||||
You can also check each application web UI for Mesos on `http://<hostname>:5050/cluster` and Spark on `http://<hostname>:8080/`.
|
||||
|
||||
|
||||
### 4. Configure Spark interpreter in Zeppelin
|
||||
|
||||
```
|
||||
export MASTER=mesos://127.0.1.1:5050
|
||||
export MESOS_NATIVE_JAVA_LIBRARY=[PATH OF libmesos.so]
|
||||
export SPARK_HOME=[PATH OF SPARK HOME]
|
||||
```
|
||||
|
||||
|
||||
Don't forget to set Spark `master` as `mesos://127.0.1.1:5050` in Zeppelin **Interpreters** setting page like below.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png" />
|
||||
|
||||
|
||||
### 5. Run Zeppelin with Spark interpreter
|
||||
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:5050/#/frameworks` and check Zeppelin application is running well or not.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" />
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Manual upgrade procedure for Zeppelin"
|
||||
description: ""
|
||||
title: "Manual Zeppelin version upgrade procedure"
|
||||
description: "This document will guide you through a procedure of manual upgrade your Apache Zeppelin instance to a newer version. Apache Zeppelin keeps backward compatibility for the notebook file format."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
|
|
@ -43,4 +43,11 @@ So, copying `notebook` and `conf` directory should be enough.
|
|||
|
||||
```
|
||||
bin/zeppelin-daemon.sh start
|
||||
```
|
||||
```
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### Upgrading from Zeppelin 0.6 to 0.7
|
||||
|
||||
- From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly.
|
||||
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Install A Zeppelin ready Virtual Machine"
|
||||
description: ""
|
||||
title: "Apache Zeppelin on Vagrant Virtual Machine"
|
||||
description: "Apache Zeppelin provides a script for running a virtual machine for development through Vagrant. The script will create a virtual machine with core dependencies pre-installed, required for developing Apache Zeppelin."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
|
|
@ -19,17 +19,17 @@ limitations under the License.
|
|||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Vagrant Virtual Machine for Apache Zeppelin
|
||||
# Apache Zeppelin on Vagrant Virtual Machine
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
## Overview
|
||||
|
||||
Apache Zeppelin distribution includes a scripts directory
|
||||
Apache Zeppelin distribution includes a script directory
|
||||
|
||||
`scripts/vagrant/zeppelin-dev`
|
||||
|
||||
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source.
|
||||
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source.
|
||||
For PySpark users, this script includes several helpful [Python Libraries](#python-extras).
|
||||
For SparkR users, this script includes several helpful [R Libraries](#r-extras).
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ By default, Vagrant will share your project directory (the directory with the Va
|
|||
Running the following commands in the guest machine should display these expected versions:
|
||||
|
||||
`node --version` should report *v0.12.7*
|
||||
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
|
||||
`mvn --version` should report *Apache Maven 3.3.9* and *Java version: 1.7.0_85*
|
||||
|
||||
The virtual machine consists of:
|
||||
|
||||
|
|
@ -96,7 +96,7 @@ The virtual machine consists of:
|
|||
- Node.js 0.12.7
|
||||
- npm 2.11.3
|
||||
- ruby 1.9.3 + rake, make and bundler (only required if building jekyll documentation)
|
||||
- Maven 3.3.3
|
||||
- Maven 3.3.9
|
||||
- Git
|
||||
- Unzip
|
||||
- libfontconfig to avoid phatomJs missing dependency issues
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Install Zeppelin to connect with existing YARN cluster"
|
||||
description: ""
|
||||
description: "This page describes how to pre-configure a bare metal node, configure Apache Zeppelin and connect it to existing YARN cluster running Hortonworks flavour of Hadoop."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Alluxio Interpreter"
|
||||
description: "Alluxio Interpreter"
|
||||
title: "Alluxio Interpreter for Apache Zeppelin"
|
||||
description: "Alluxio is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Alluxio Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,23 @@
|
|||
---
|
||||
layout: page
|
||||
title: "BigQuery Interpreter"
|
||||
description: ""
|
||||
title: "BigQuery Interpreter for Apache Zeppelin"
|
||||
description: "BigQuery is a highly scalable no-ops data warehouse in the Google Cloud Platform."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# BigQuery Interpreter for Apache Zeppelin
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Cassandra Interpreter"
|
||||
description: "Cassandra Interpreter"
|
||||
title: "Cassandra CQL Interpreter for Apache Zeppelin"
|
||||
description: "Apache Cassandra database is the right choice when you need scalability and high availability without compromising performance."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Cassandra CQL Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Elasticsearch Interpreter"
|
||||
description: ""
|
||||
title: "Elasticsearch Interpreter for Apache Zeppelin"
|
||||
description: "Elasticsearch is a highly scalable open-source full-text search and analytics engine."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Elasticsearch Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Flink Interpreter"
|
||||
description: ""
|
||||
title: "Flink Interpreter for Apache Zeppelin"
|
||||
description: "Apache Flink is an open source platform for distributed stream and batch data processing."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Flink interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Geode OQL Interpreter"
|
||||
description: ""
|
||||
title: "Geode/Gemfire OQL Interpreter for Apache Zeppelin"
|
||||
description: "Apache Geode (incubating) provides a database-like consistency model, reliable transaction processing and a shared-nothing architecture to maintain very low latency performance with high concurrency processing."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Geode/Gemfire OQL Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "HBase Shell Interpreter"
|
||||
description: ""
|
||||
title: "HBase Shell Interpreter for Apache Zeppelin"
|
||||
description: "HBase Shell is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# HBase Shell Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "HDFS File System Interpreter"
|
||||
description: ""
|
||||
title: "HDFS File System Interpreter for Apache Zeppelin"
|
||||
description: "Hadoop File System is a distributed, fault tolerant file system part of the hadoop project and is often used as storage for distributed processing engines like Hadoop MapReduce and Apache Spark or underlying file systems like Alluxio."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# HDFS File System Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Hive Interpreter"
|
||||
description: ""
|
||||
title: "Hive Interpreter for Apache Zeppelin"
|
||||
description: "Apache Hive data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Hive Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Ignite Interpreter"
|
||||
description: "Ignite user guide"
|
||||
title: "Ignite Interpreter for Apache Zeppelin"
|
||||
description: "Apache Ignite in-memory Data Fabric is a high-performance, integrated and distributed in-memory platform for computing and transacting on large-scale data sets in real-time, orders of magnitude faster than possible with traditional disk-based or flash technologies."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Ignite Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Generic JDBC Interpreter"
|
||||
description: "JDBC user guide"
|
||||
title: "Generic JDBC Interpreter for Apache Zeppelin"
|
||||
description: "Generic JDBC Interpreter lets you create a JDBC connection to any data source. You can use Postgres, MySql, MariaDB, Redshift, Apache Hive, Apache Phoenix, Apache Drill and Apache Tajo using JDBC interpreter."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
|
||||
# Generic JDBC Interpreter for Apache Zeppelin
|
||||
# Generic JDBC Interpreter for Apache Zeppelin
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Lens Interpreter"
|
||||
description: "Lens user guide"
|
||||
title: "Lens Interpreter for Apache Zeppelin"
|
||||
description: "Apache Lens provides an Unified Analytics interface. Lens aims to cut the Data Analytics silos by providing a single view of data across multiple tiered data stores and optimal execution environment for the analytical query. It seamlessly integrates Hadoop with traditional data warehouses to appear like one."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Lens Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Livy Interpreter"
|
||||
description: ""
|
||||
title: "Livy Interpreter for Apache Zeppelin"
|
||||
description: "Livy is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Livy Interpreter for Apache Zeppelin
|
||||
|
|
@ -30,7 +43,7 @@ We added some common configurations for spark, and you can set any configuration
|
|||
This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties.
|
||||
And instead of starting property with `spark.` it should be replaced with `livy.spark.`.
|
||||
Example: `spark.master` to `livy.spark.master`
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Property</th>
|
||||
|
|
@ -50,7 +63,7 @@ Example: `spark.master` to `livy.spark.master`
|
|||
<tr>
|
||||
<td>zeppelin.livy.spark.maxResult</td>
|
||||
<td>1000</td>
|
||||
<td>Max number of SparkSQL result to display.</td>
|
||||
<td>Max number of Spark SQL result to display.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>livy.spark.driver.cores</td>
|
||||
|
|
@ -102,8 +115,31 @@ Example: `spark.master` to `livy.spark.master`
|
|||
<td></td>
|
||||
<td>Upper bound for the number of executors.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>livy.spark.jars.packages</td>
|
||||
<td></td>
|
||||
<td>Adding extra libraries to livy interpreter</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Adding External libraries
|
||||
You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version.
|
||||
|
||||
Example
|
||||
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Property</th>
|
||||
<th>Example</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>livy.spark.jars.packages</td>
|
||||
<td>io.spray:spray-json_2.10:1.3.1</td>
|
||||
<td>Adding extra libraries to livy interpreter</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## How to use
|
||||
Basically, you can use
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Markdown Interpreter"
|
||||
description: "Markdown Interpreter"
|
||||
title: "Markdown Interpreter for Apache Zeppelin"
|
||||
description: "Markdown is a plain text formatting syntax designed so that it can be converted to HTML. Apache Zeppelin uses markdown4j."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Markdown Interpreter for Apache Zeppelin
|
||||
|
|
@ -12,7 +25,7 @@ group: interpreter
|
|||
|
||||
## Overview
|
||||
[Markdown](http://daringfireball.net/projects/markdown/) is a plain text formatting syntax designed so that it can be converted to HTML.
|
||||
Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/).
|
||||
Apache Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/).
|
||||
In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to invoke the Markdown interpreter and generate static html from Markdown plain text.
|
||||
|
||||
In Zeppelin, Markdown interpreter is enabled by default.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "PostgreSQL and HAWQ Interpreter"
|
||||
description: ""
|
||||
title: "PostgreSQL, Apache HAWQ (incubating) Interpreter for Apache Zeppelin"
|
||||
description: "Apache Zeppelin supports PostgreSQL, Apache HAWQ(incubating) and Greenplum SQL data processing engines."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# PostgreSQL, Apache HAWQ (incubating) Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Python Interpreter"
|
||||
description: "Python Interpreter"
|
||||
title: "Python 2 & 3 Interpreter for Apache Zeppelin"
|
||||
description: "Python is a programming language that lets you work quickly and integrate systems more effectively."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Python 2 & 3 Interpreter for Apache Zeppelin
|
||||
|
|
@ -74,7 +87,7 @@ print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
|
|||
|
||||
## Matplotlib integration
|
||||
The python interpreter can display matplotlib graph with the function `z.show()`.
|
||||
You need to have matplotlib module installed and a XServer running to use this functionality !
|
||||
You need to have matplotlib module installed and a XServer running to use this functionality!
|
||||
|
||||
```python
|
||||
%python
|
||||
|
|
@ -84,12 +97,12 @@ plt.figure()
|
|||
z.show(plt)
|
||||
plt.close()
|
||||
```
|
||||
z.show function can take optional parameters to adapt graph width and height
|
||||
The `z.show()` function can take optional parameters to adapt graph dimensions (width and height) as well as output format (png or optionally svg).
|
||||
|
||||
```python
|
||||
%python
|
||||
z.show(plt, width='50px')
|
||||
z.show(plt, height='150px')
|
||||
z.show(plt, height='150px', fmt='svg')
|
||||
```
|
||||
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/pythonMatplotlib.png" />
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "R Interpreter"
|
||||
description: ""
|
||||
title: "R Interpreter for Apache Zeppelin"
|
||||
description: "R is a free software environment for statistical computing and graphics."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# R Interpreter for Apache Zeppelin
|
||||
|
|
@ -104,7 +117,7 @@ And vice versa:
|
|||
|
||||
* The `knitr` environment is persistent. If you run a chunk from Zeppelin that changes a variable, then run the same chunk again, the variable has already been changed. Use immutable variables.
|
||||
|
||||
* (Note that `%spark.r` and `$r` are two different ways of calling the same interpreter, as are `%spark.knitr` and `%knitr`. By default, Zeppelin puts the R interpreters in the `%spark.` Interpreter Group.
|
||||
* (Note that `%spark.r` and `%r` are two different ways of calling the same interpreter, as are `%spark.knitr` and `%knitr`. By default, Zeppelin puts the R interpreters in the `%spark.` Interpreter Group.
|
||||
|
||||
* Using the `%r` interpreter, if you return a data.frame, HTML, or an image, it will dominate the result. So if you execute three commands, and one is `hist()`, all you will see is the histogram, not the results of the other commands. This is a Zeppelin limitation.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Scalding Interpreter"
|
||||
description: ""
|
||||
title: "Scalding Interpreter for Apache Zeppelin"
|
||||
description: "Scalding is an open source Scala library for writing MapReduce jobs."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Scalding Interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,9 +1,22 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Shell Interpreter"
|
||||
description: "Shell Interpreter"
|
||||
title: "Shell interpreter for Apache Zeppelin"
|
||||
description: "Shell interpreter uses Apache Commons Exec to execute external processes."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Shell interpreter for Apache Zeppelin
|
||||
|
|
|
|||
|
|
@ -1,11 +1,23 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Spark Interpreter Group"
|
||||
description: ""
|
||||
title: "Apache Spark Interpreter for Apache Zeppelin"
|
||||
description: "Apache Spark is a fast and general-purpose cluster computing system. It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs."
|
||||
group: interpreter
|
||||
---
|
||||
{% include JB/setup %}
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Spark Interpreter for Apache Zeppelin
|
||||
|
||||
|
|
@ -79,7 +91,7 @@ You can also set other Spark properties which are not listed in the table. For a
|
|||
</tr>
|
||||
<tr>
|
||||
<td>spark.executor.memory </td>
|
||||
<td>512m</td>
|
||||
<td>1g</td>
|
||||
<td>Executor memory per worker instance. <br/> ex) 512m, 32g</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -105,7 +117,7 @@ You can also set other Spark properties which are not listed in the table. For a
|
|||
<tr>
|
||||
<td>zeppelin.spark.maxResult</td>
|
||||
<td>1000</td>
|
||||
<td>Max number of SparkSQL result to display.</td>
|
||||
<td>Max number of Spark SQL result to display.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>zeppelin.spark.printREPLOutput</td>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Dependency Management"
|
||||
description: ""
|
||||
title: "Dependency Management for Apache Spark Interpreter"
|
||||
description: "Include external libraries to Apache Spark Interpreter by setting dependencies in interpreter menu."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Dynamic Form"
|
||||
description: ""
|
||||
title: "Dynamic Form in Apache Zeppelin"
|
||||
description: "Apache Zeppelin dynamically creates input forms. Depending on language backend, there're two different ways to create dynamic form."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
@ -28,7 +28,7 @@ Custom language backend can select which type of form creation it wants to use.
|
|||
|
||||
## Using form Templates
|
||||
|
||||
This mode creates form using simple template language. It's simple and easy to use. For example Markdown, Shell, SparkSql language backend uses it.
|
||||
This mode creates form using simple template language. It's simple and easy to use. For example Markdown, Shell, Spark SQL language backend uses it.
|
||||
|
||||
### Text input form
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Dynamic Interpreter Loading"
|
||||
description: ""
|
||||
title: "Dynamic Interpreter Loading using REST API"
|
||||
description: "Apache Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this page, we will introduce dynamic interpreter loading using REST API."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
@ -23,7 +23,7 @@ limitations under the License.
|
|||
|
||||
<div id="toc"></div>
|
||||
|
||||
Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal).
|
||||
Apache Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal).
|
||||
Before we start, if you are not familiar with the concept of **Zeppelin interpreter**, you can check out [Overview of Zeppelin interpreter](../manual/interpreters.html) first.
|
||||
|
||||
## Overview
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Interpreter Installation"
|
||||
description: ""
|
||||
title: "Interpreter Installation in Netinst Binary Package"
|
||||
description: "Apache Zeppelin provides Interpreter Installation mechanism for whom downloaded Zeppelin netinst binary package, or just want to install another 3rd party interpreters."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
@ -84,6 +84,15 @@ If you install one of these interpreters only with `--name` option, installer wi
|
|||
./bin/install-interpreter.sh --name flink --artifact org.apache.zeppelin:zeppelin-flink_2.10:0.6.1
|
||||
```
|
||||
|
||||
#### Install Spark interpreter built with Scala 2.10
|
||||
Spark distribution package has been built with Scala 2.10 until 1.6.2. If you have `SPARK_HOME` set pointing to Spark version ealier than 2.0.0, you need to download Spark interpreter packaged with Scala 2.10. To do so, use follow command:
|
||||
|
||||
```
|
||||
rm -rf ./interpreter/spark
|
||||
./bin/install-interpreter.sh --name spark --artifact org.apache.zeppelin:zeppelin-spark_2.10:0.6.1
|
||||
```
|
||||
|
||||
<br />
|
||||
Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Interpreters"
|
||||
description: ""
|
||||
title: "Interpreters in Apache Zeppelin"
|
||||
description: "This document explains about the role of interpreters, interpreters group and interpreter settings in Apache Zeppelin. The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
@ -27,7 +27,7 @@ limitations under the License.
|
|||
|
||||
In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin.
|
||||
The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin.
|
||||
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, JDBC, Markdown, Shell and so on.
|
||||
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), Spark SQL, JDBC, Markdown, Shell and so on.
|
||||
|
||||
## What is Zeppelin interpreter?
|
||||
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use Scala code in Zeppelin, you need `%spark` interpreter.
|
||||
|
|
@ -51,7 +51,7 @@ Each notebook can be bound to multiple Interpreter Settings using setting icon o
|
|||
|
||||
## What is interpreter group?
|
||||
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
|
||||
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, SparkSQL and the dependency loader.
|
||||
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, Spark SQL and the dependency loader.
|
||||
|
||||
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Notebook as Homepage"
|
||||
description: ""
|
||||
title: "Customize Apache Zeppelin homepage"
|
||||
description: "Apache Zeppelin allows you to use one of the notebooks you create as your Zeppelin Homepage. With that you can brand your Zeppelin installation, adjust the instruction to your users needs and even translate to other languages."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Publish your Paragraph"
|
||||
description: ""
|
||||
title: "How can you publish your paragraph"
|
||||
description: "Apache Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website."
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Please contribute"
|
||||
description: ""
|
||||
group: development
|
||||
title:
|
||||
description:
|
||||
group:
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Explore Apache Zeppelin UI"
|
||||
description: "Description of Zeppelin UI Layout"
|
||||
description: "If you are new to Apache Zeppelin, this document will guide you about the basic components of Zeppelin one by one."
|
||||
group: quickstart
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,6 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Explore Apache Zeppelin UI
|
||||
|
||||
<div id="toc"></div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Tutorial"
|
||||
description: "Tutorial is valid for Spark 1.3 and higher"
|
||||
title: "Apache Zeppelin Tutorial"
|
||||
description: "This tutorial page contains a short walk-through tutorial that uses Apache Spark backend. Please note that this tutorial is valid for Spark 1.3 and higher."
|
||||
group: quickstart
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,6 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Zeppelin Tutorial
|
||||
|
||||
<div id="toc"></div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Configuration REST API"
|
||||
description: ""
|
||||
title: "Apache Zeppelin Configuration REST API"
|
||||
description: "This page contains Apache Zeppelin Configuration REST API information."
|
||||
group: rest-api
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Credentials REST API"
|
||||
description: ""
|
||||
title: "Apache Zeppelin Credential REST API"
|
||||
description: "This page contains Apache Zeppelin Credential REST API information."
|
||||
group: rest-api
|
||||
---
|
||||
<!--
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Interpreter REST API"
|
||||
description: ""
|
||||
title: "Apache Zeppelin Interpreter REST API"
|
||||
description: "This page contains Apache Zeppelin Interpreter REST API information."
|
||||
group: rest-api
|
||||
---
|
||||
<!--
|
||||
|
|
@ -75,7 +75,7 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"className": "org.apache.zeppelin.spark.SparkInterpreter",
|
||||
"properties": {
|
||||
"spark.executor.memory": {
|
||||
"defaultValue": "512m",
|
||||
"defaultValue": "1g",
|
||||
"description": "Executor memory per worker instance. ex) 512m, 32g"
|
||||
},
|
||||
"spark.cores.max": {
|
||||
|
|
@ -92,7 +92,7 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"properties": {
|
||||
"zeppelin.spark.maxResult": {
|
||||
"defaultValue": "1000",
|
||||
"description": "Max number of SparkSQL result to display."
|
||||
"description": "Max number of Spark SQL result to display."
|
||||
}
|
||||
},
|
||||
"path": "/zeppelin/interpreter/spark"
|
||||
|
|
@ -154,7 +154,7 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"group": "spark",
|
||||
"properties": {
|
||||
"spark.cores.max": "",
|
||||
"spark.executor.memory": "512m",
|
||||
"spark.executor.memory": "1g",
|
||||
},
|
||||
"interpreterGroup": [
|
||||
{
|
||||
|
|
@ -198,7 +198,10 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
<td>
|
||||
400 if the input json is empty <br/>
|
||||
500 for any other errors
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Sample JSON input</td>
|
||||
|
|
@ -219,7 +222,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -249,7 +254,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -298,7 +305,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -328,7 +337,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -460,4 +471,4 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
<td> 500 </td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Notebook REST API"
|
||||
description: ""
|
||||
title: "Apache Zeppelin Notebook REST API"
|
||||
description: "This page contains Apache Zeppelin Notebook REST API information."
|
||||
group: rest-api
|
||||
---
|
||||
<!--
|
||||
|
|
@ -298,7 +298,10 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method runs all paragraphs in the given notebook id.
|
||||
<td>
|
||||
This ```POST``` method runs all paragraphs in the given notebook id. <br />
|
||||
If you can not find Notebook id 404 returns.
|
||||
If there is a problem with the interpreter returns a 412 error.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -311,12 +314,29 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
<td> 404 or 412</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response </td>
|
||||
<td><pre>{"status": "OK"}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON error response </td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
"status": "NOT_FOUND",
|
||||
"message": "note not found."
|
||||
}
|
||||
</pre><br />
|
||||
<pre>
|
||||
{
|
||||
"status": "PRECONDITION_FAILED",
|
||||
"message": "paragraph_1469771130099_-278315611 Not selected or Invalid Interpreter bind"
|
||||
}
|
||||
</pre>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
|
|
@ -393,12 +413,49 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</table>
|
||||
|
||||
<br/>
|
||||
### Run a paragraph
|
||||
### Get the status of a single paragraph
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
|
||||
<td>This ```GET``` method gets the status of a single paragraph by the given notebook and paragraph id.
|
||||
The body field of the returned JSON contains of the array that compose of the paragraph id, paragraph status, paragraph finish date, paragraph started date.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Success code</td>
|
||||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response </td>
|
||||
<td><pre>
|
||||
{
|
||||
"status": "OK",
|
||||
"body": {
|
||||
"id":"20151121-212654\_766735423",
|
||||
"status":"FINISHED",
|
||||
"finished":"Tue Nov 24 14:21:40 KST 2015",
|
||||
"started":"Tue Nov 24 14:21:39 KST 2015"
|
||||
}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Run a paragraph asynchronously
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method runs the paragraph asynchronously by given notebook and paragraph id. This API always return SUCCESS even if the execution of the paragraph fails later because the API is asynchronous
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -430,6 +487,56 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Run a paragraph synchronously
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td> This ```POST``` method runs the paragraph synchronously by given notebook and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Success code</td>
|
||||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON input (optional, only needed when if you want to update dynamic form's value) </td>
|
||||
<td><pre>
|
||||
{
|
||||
"name": "name of new notebook",
|
||||
"params": {
|
||||
"formLabel1": "value1",
|
||||
"formLabel2": "value2"
|
||||
}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response </td>
|
||||
<td><pre>{"status": "OK"}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON error </td>
|
||||
<td><pre>
|
||||
{
|
||||
"status": "INTERNAL\_SERVER\_ERROR",
|
||||
"body": {
|
||||
"code": "ERROR",
|
||||
"type": "TEXT",
|
||||
"msg": "bash: -c: line 0: unexpected EOF while looking for matching ``'\nbash: -c: line 1: syntax error: unexpected end of file\nExitValue: 2"
|
||||
}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Stop a paragraph
|
||||
<table class="table-configuration">
|
||||
|
|
@ -865,4 +972,3 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Screenshots"
|
||||
description: ""
|
||||
title:
|
||||
description:
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -21,7 +21,7 @@ limitations under the License.
|
|||
<div class="row">
|
||||
<div class="col-md-3">
|
||||
<a href="assets/themes/zeppelin/img/screenshots/sparksql.png"><img class="thumbnail" src="assets/themes/zeppelin/img/screenshots/sparksql.png" /></a>
|
||||
<center>SparkSQL with inline visualization</center>
|
||||
<center>Spark SQL with inline visualization</center>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<a href="assets/themes/zeppelin/img/screenshots/spark.png"><img class="thumbnail" src="assets/themes/zeppelin/img/screenshots/spark.png" /></a>
|
||||
|
|
|
|||
43
docs/search.md
Normal file
43
docs/search.md
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
---
|
||||
layout: page
|
||||
title:
|
||||
description:
|
||||
group:
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
<div id="home-search" class="home">
|
||||
|
||||
<span id="search">
|
||||
<p><i class="glyphicon glyphicon-search"></i> Search Docs</p>
|
||||
</span>
|
||||
<br/>
|
||||
|
||||
<form role="search" action="{{BASE_PATH}}/search.html" id="site_search">
|
||||
<div class="input-group" id="search-container">
|
||||
<input type="text" class="form-control" size="16px" name="q" placeholder="Search all pages" id="search_box">
|
||||
<span class="input-group-btn">
|
||||
<button type="reset" class="btn btn-default">
|
||||
<i class="glyphicon glyphicon-remove" style="color:#777"></i>
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<br/><br/>
|
||||
|
||||
<div id="search_results"></div>
|
||||
</div>
|
||||
17
docs/search_data.json
Normal file
17
docs/search_data.json
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
layout: null
|
||||
---
|
||||
{
|
||||
{% for page in site.pages %}{% if page.title != nil %}
|
||||
|
||||
"{{ page.url | slugify }}": {
|
||||
"title": "{{ page.title | xml_escape }}",
|
||||
"content" : "{{page.content | strip_html | strip_newlines | escape | remove: "\"}}",
|
||||
"url": " {{ page.url | xml_escape }}",
|
||||
"group": "{{ page.group }}",
|
||||
"excerpt": {{ page.description | strip_html | truncatewords: 40 | jsonify }}
|
||||
}
|
||||
{% unless forloop.last %},{% endunless %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Authentication for NGINX"
|
||||
description: "Authentication for NGINX"
|
||||
description: "There are multiple ways to enable authentication in Apache Zeppelin. This page describes HTTP basic auth using NGINX."
|
||||
group: security
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,11 +17,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Authentication for NGINX
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
Authentication is company-specific.
|
||||
There are multiple ways to enable authentication in Apache Zeppelin. This page describes HTTP basic auth using NGINX.
|
||||
One option is to use [Basic Access Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
|
||||
|
||||
## HTTP Basic Authentication using NGINX
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Data Source Authorization"
|
||||
description: "Data Source Authorization"
|
||||
title: "Data Source Authorization in Apache Zeppelin"
|
||||
description: "Apache Zeppelin supports protected data sources. In case of a MySql database, every users can set up their own credentials to access it."
|
||||
group: security
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,6 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Data Source Authorization in Apache Zeppelin
|
||||
|
||||
<div id="toc"></div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Notebook Authorization"
|
||||
description: "Notebook Authorization"
|
||||
title: "Notebook Authorization in Apache Zeppelin"
|
||||
description: "This page will guide you how you can set the permission for Zeppelin notebooks. This document assumes that Apache Shiro authentication was set up."
|
||||
group: security
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,6 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Zeppelin Notebook Authorization
|
||||
|
||||
<div id="toc"></div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Shiro Security for Apache Zeppelin"
|
||||
description: ""
|
||||
title: "Apache Shiro Authentication for Apache Zeppelin"
|
||||
description: "Apache Shiro is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. This document explains step by step how Shiro can be used for Zeppelin notebook authentication."
|
||||
group: security
|
||||
---
|
||||
<!--
|
||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
|||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Shiro authentication for Apache Zeppelin
|
||||
# Apache Shiro authentication for Apache Zeppelin
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
|
|
@ -105,6 +105,59 @@ finance = *
|
|||
group1 = *
|
||||
```
|
||||
|
||||
## Configure Realm (optional)
|
||||
Realms are responsible for authentication and authorization in Apache Zeppelin. By default, Apache Zeppelin uses [IniRealm](https://shiro.apache.org/static/latest/apidocs/org/apache/shiro/realm/text/IniRealm.html) (users and groups are configurable in `conf/shiro.ini` file under `[user]` and `[group]` section). You can also leverage Shiro Realms like [JndiLdapRealm](https://shiro.apache.org/static/latest/apidocs/org/apache/shiro/realm/ldap/JndiLdapRealm.html), [JdbcRealm](https://shiro.apache.org/static/latest/apidocs/org/apache/shiro/realm/jdbc/JdbcRealm.html) or create [our own](https://shiro.apache.org/static/latest/apidocs/org/apache/shiro/realm/AuthorizingRealm.html).
|
||||
To learn more about Apache Shiro Realm, please check [this documentation](http://shiro.apache.org/realm.html).
|
||||
|
||||
We also provide community custom Realms.
|
||||
|
||||
### Active Directory
|
||||
|
||||
```
|
||||
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
|
||||
activeDirectoryRealm.systemUsername = userNameA
|
||||
activeDirectoryRealm.systemPassword = passwordA
|
||||
activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks
|
||||
activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
||||
activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
|
||||
activeDirectoryRealm.authorizationCachingEnabled = false
|
||||
```
|
||||
|
||||
|
||||
Also instead of specifying systemPassword in clear text in shiro.ini administrator can choose to specify the same in "hadoop credential".
|
||||
Create a keystore file using the hadoop credential commandline, for this the hadoop commons should be in the classpath
|
||||
`hadoop credential create activeDirectoryRealm.systempassword -provider jceks://file/user/zeppelin/conf/zeppelin.jceks`
|
||||
|
||||
Change the following values in the Shiro.ini file, and uncomment the line:
|
||||
`activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks`
|
||||
|
||||
### LDAP
|
||||
|
||||
```
|
||||
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
|
||||
# search base for ldap groups (only relevant for LdapGroupRealm):
|
||||
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
|
||||
ldapRealm.contextFactory.url = ldap://ldap.test.com:389
|
||||
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
|
||||
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
|
||||
```
|
||||
|
||||
### ZeppelinHub
|
||||
[ZeppelinHub](https://www.zeppelinhub.com) is a service that synchronize your Apache Zeppelin notebooks and enables you to collaborate easily.
|
||||
|
||||
To enable login with your ZeppelinHub credential, apply the following change in `conf/shiro.ini` under `[main]` section.
|
||||
|
||||
```
|
||||
### A sample for configuring ZeppelinHub Realm
|
||||
zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
|
||||
## Url of ZeppelinHub
|
||||
zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
|
||||
securityManager.realms = $zeppelinHubRealm
|
||||
```
|
||||
|
||||
> Note: ZeppelinHub is not releated to apache Zeppelin project.
|
||||
|
||||
## Secure your Zeppelin information (optional)
|
||||
By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
|
||||
Sometimes you might want to hide these information for your use case.
|
||||
|
|
@ -123,3 +176,4 @@ If you want to grant this permission to other users, you can change **roles[ ]**
|
|||
|
||||
<br/>
|
||||
> **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file. This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Storage"
|
||||
description: "Notebook Storage option for Zeppelin"
|
||||
title: "Notebook Storage for Apache Zeppelin"
|
||||
description: Apache Zeppelin has a pluggable notebook storage mechanism controlled by zeppelin.notebook.storage configuration option with multiple implementations."
|
||||
group: storage
|
||||
---
|
||||
<!--
|
||||
|
|
@ -17,6 +17,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Notebook storage options for Apache Zeppelin
|
||||
|
||||
<div id="toc"></div>
|
||||
|
|
|
|||
|
|
@ -120,21 +120,18 @@
|
|||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
<version>${scala.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-compiler</artifactId>
|
||||
<version>${scala.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-reflect</artifactId>
|
||||
<version>${scala.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -146,24 +143,6 @@
|
|||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/.idea/</exclude>
|
||||
<exclude>**/*.iml</exclude>
|
||||
<exclude>.gitignore</exclude>
|
||||
<exclude>**/.settings/*</exclude>
|
||||
<exclude>**/.classpath</exclude>
|
||||
<exclude>**/.project</exclude>
|
||||
<exclude>**/target/**</exclude>
|
||||
<exclude>**/README.md</exclude>
|
||||
<exclude>**/interpreter-setting.json</exclude>
|
||||
<exclude>dependency-reduced-pom.xml</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- Scala Compiler -->
|
||||
<plugin>
|
||||
|
|
@ -319,7 +298,6 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.4</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
|
|
@ -339,10 +317,11 @@
|
|||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
|
|
|
|||
|
|
@ -54,23 +54,15 @@ import java.util.Properties;
|
|||
* zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
|
||||
*/
|
||||
public class HbaseInterpreter extends Interpreter {
|
||||
public static final String HBASE_HOME = "hbase.home";
|
||||
public static final String HBASE_RUBY_SRC = "hbase.ruby.sources";
|
||||
public static final String HBASE_TEST_MODE = "zeppelin.hbase.test.mode";
|
||||
|
||||
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
|
||||
private ScriptingContainer scriptingContainer;
|
||||
|
||||
private StringWriter writer;
|
||||
|
||||
static {
|
||||
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
|
||||
new InterpreterPropertyBuilder()
|
||||
.add("hbase.home",
|
||||
getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
|
||||
"Installation directory of HBase")
|
||||
.add("hbase.ruby.sources", "lib/ruby",
|
||||
"Path to Ruby scripts relative to 'hbase.home'")
|
||||
.add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
|
||||
.build());
|
||||
}
|
||||
|
||||
public HbaseInterpreter(Properties property) {
|
||||
super(property);
|
||||
}
|
||||
|
|
@ -81,9 +73,9 @@ public class HbaseInterpreter extends Interpreter {
|
|||
this.writer = new StringWriter();
|
||||
scriptingContainer.setOutput(this.writer);
|
||||
|
||||
if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
|
||||
String hbase_home = getProperty("hbase.home");
|
||||
String ruby_src = getProperty("hbase.ruby.sources");
|
||||
if (!Boolean.parseBoolean(getProperty(HBASE_TEST_MODE))) {
|
||||
String hbase_home = getProperty(HBASE_HOME);
|
||||
String ruby_src = getProperty(HBASE_RUBY_SRC);
|
||||
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
|
||||
|
||||
logger.info("Home:" + hbase_home);
|
||||
|
|
@ -98,7 +90,7 @@ public class HbaseInterpreter extends Interpreter {
|
|||
logger.info("Absolute Ruby Source:" + abs_ruby_src.toString());
|
||||
// hirb.rb:41 requires the following system property to be set.
|
||||
Properties sysProps = System.getProperties();
|
||||
sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString());
|
||||
sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString());
|
||||
|
||||
Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb");
|
||||
try {
|
||||
|
|
|
|||
25
hbase/src/main/resources/interpreter-setting.json
Normal file
25
hbase/src/main/resources/interpreter-setting.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[
|
||||
{
|
||||
"group": "hbase",
|
||||
"name": "hbase",
|
||||
"className": "org.apache.zeppelin.hbase.HbaseInterpreter",
|
||||
"properties": {
|
||||
"hbase.home": {
|
||||
"envName": "HBASE_HOME",
|
||||
"propertyName": "hbase.home",
|
||||
"defaultValue": "/usr/lib/hbase/",
|
||||
"description": "Installation directory of HBase"
|
||||
},
|
||||
"hbase.ruby.sources": {
|
||||
"propertyName": "hbase.ruby.sources",
|
||||
"defaultValue": "lib/ruby",
|
||||
"description": "Path to Ruby scripts relative to 'hbase.home'"
|
||||
},
|
||||
"zeppelin.hbase.test.mode": {
|
||||
"propertyName": "zeppelin.hbase.test.mode",
|
||||
"defaultValue": "false",
|
||||
"description": "Disable checks for unit and manual tests"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
<name>Zeppelin: Apache Ignite interpreter</name>
|
||||
|
||||
<properties>
|
||||
<ignite.version>1.5.0.final</ignite.version>
|
||||
<ignite.version>1.7.0</ignite.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
@ -116,8 +116,8 @@
|
|||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
package org.apache.zeppelin.jdbc;
|
||||
|
||||
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.io.IOException;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.sql.Connection;
|
||||
|
|
@ -418,11 +419,11 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
} catch (Exception e) {
|
||||
logger.error("Cannot run " + sql, e);
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
stringBuilder.append(e.getMessage()).append("\n");
|
||||
stringBuilder.append(e.getClass().toString()).append("\n");
|
||||
stringBuilder.append(StringUtils.join(e.getStackTrace(), "\n"));
|
||||
return new InterpreterResult(Code.ERROR, stringBuilder.toString());
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintStream ps = new PrintStream(baos);
|
||||
e.printStackTrace(ps);
|
||||
String errorMsg = new String(baos.toByteArray(), StandardCharsets.UTF_8);
|
||||
return new InterpreterResult(Code.ERROR, errorMsg);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -495,7 +496,8 @@ public class JDBCInterpreter extends Interpreter {
|
|||
public Scheduler getScheduler() {
|
||||
String schedulerName = JDBCInterpreter.class.getName() + this.hashCode();
|
||||
return isConcurrentExecution() ?
|
||||
SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName, 10)
|
||||
SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName,
|
||||
getMaxConcurrentConnection())
|
||||
: SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName);
|
||||
}
|
||||
|
||||
|
|
|
|||
19
licenses/LICENSE-lunrjs-0.7.1
Normal file
19
licenses/LICENSE-lunrjs-0.7.1
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
Copyright (C) 2013 by Oliver Nightingale
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
|
@ -81,6 +81,11 @@
|
|||
"propertyName": "zeppelin.livy.keytab",
|
||||
"defaultValue": "",
|
||||
"description": "Kerberos keytab to authenticate livy"
|
||||
},
|
||||
"livy.spark.jars.packages": {
|
||||
"propertyName": "livy.spark.jars.packages",
|
||||
"defaultValue": "",
|
||||
"description": "Adding extra libraries to livy interpreter"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -93,7 +98,7 @@
|
|||
"envName": "ZEPPELIN_LIVY_MAXRESULT",
|
||||
"propertyName": "zeppelin.livy.spark.sql.maxResult",
|
||||
"defaultValue": "1000",
|
||||
"description": "Max number of SparkSQL result to display."
|
||||
"description": "Max number of Spark SQL result to display."
|
||||
},
|
||||
"zeppelin.livy.concurrentSQL": {
|
||||
"propertyName": "zeppelin.livy.concurrentSQL",
|
||||
|
|
@ -116,4 +121,4 @@
|
|||
"properties": {
|
||||
}
|
||||
}
|
||||
]
|
||||
]
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
263
pom.xml
263
pom.xml
|
|
@ -211,11 +211,6 @@
|
|||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
|
|
@ -411,122 +406,6 @@
|
|||
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
<version>0.11</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/*.keywords</exclude>
|
||||
<exclude>reports/**</exclude>
|
||||
<exclude>**/.idea/</exclude>
|
||||
<exclude>**/*.iml</exclude>
|
||||
<exclude>.git/</exclude>
|
||||
<exclude>.github/*</exclude>
|
||||
<exclude>.gitignore</exclude>
|
||||
<exclude>.repository/</exclude>
|
||||
<exclude>.Rhistory</exclude>
|
||||
<exclude>**/*.diff</exclude>
|
||||
<exclude>**/*.patch</exclude>
|
||||
<exclude>**/*.avsc</exclude>
|
||||
<exclude>**/*.avro</exclude>
|
||||
<exclude>**/*.log</exclude>
|
||||
<exclude>**/test/resources/**</exclude>
|
||||
<exclude>**/.settings/*</exclude>
|
||||
<exclude>**/.classpath</exclude>
|
||||
<exclude>**/.project</exclude>
|
||||
<exclude>**/target/**</exclude>
|
||||
<exclude>**/derby.log</exclude>
|
||||
<exclude>**/metastore_db/</exclude>
|
||||
<exclude>**/logs/**</exclude>
|
||||
<exclude>**/run/**</exclude>
|
||||
<exclude>**/interpreter/**</exclude>
|
||||
<exclude>**/local-repo/**</exclude>
|
||||
<exclude>**/null/**</exclude>
|
||||
<exclude>**/notebook/**</exclude>
|
||||
<exclude>_tools/site/css/*</exclude>
|
||||
<exclude>**/README.md</exclude>
|
||||
<exclude>DEPENDENCIES</exclude>
|
||||
<exclude>DEPLOY.md</exclude>
|
||||
<exclude>CONTRIBUTING.md</exclude>
|
||||
<exclude>STYLE.md</exclude>
|
||||
<exclude>Roadmap.md</exclude>
|
||||
<exclude>**/licenses/**</exclude>
|
||||
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
|
||||
<exclude>conf/interpreter.json</exclude>
|
||||
<exclude>conf/notebook-authorization.json</exclude>
|
||||
<exclude>conf/credentials.json</exclude>
|
||||
<exclude>conf/zeppelin-env.sh</exclude>
|
||||
<exclude>spark-*-bin*/**</exclude>
|
||||
<exclude>.spark-dist/**</exclude>
|
||||
<exclude>**/interpreter-setting.json</exclude>
|
||||
<exclude>**/constants.json</exclude>
|
||||
|
||||
<!-- bundled from bootstrap -->
|
||||
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
|
||||
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
|
||||
|
||||
<!-- bundled from jekyll-bootstrap -->
|
||||
<exclude>docs/404.html</exclude>
|
||||
<exclude>docs/_config.yml</exclude>
|
||||
<exclude>docs/_includes/JB/**</exclude>
|
||||
<exclude>docs/_layouts/**</exclude>
|
||||
<exclude>docs/_plugins/**</exclude>
|
||||
<exclude>docs/atom.xml</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
|
||||
<exclude>docs/Rakefile</exclude>
|
||||
<exclude>docs/rss.xml</exclude>
|
||||
<exclude>docs/sitemap.txt</exclude>
|
||||
<exclude>**/dependency-reduced-pom.xml</exclude>
|
||||
|
||||
<!-- bundled from anchor -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
|
||||
|
||||
<!-- bundled from toc -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
|
||||
|
||||
<!-- bundled from jekyll -->
|
||||
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
|
||||
|
||||
<!-- docs (website) build target dir -->
|
||||
<exclude>docs/_site/**</exclude>
|
||||
<exclude>docs/Gemfile.lock</exclude>
|
||||
|
||||
<!-- compiled R packages (binaries) -->
|
||||
<exclude>R/lib/**</exclude>
|
||||
|
||||
<!--R-related files with alternative licenses-->
|
||||
<exclude>r/R/rzeppelin/R/globals.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/common.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
|
||||
<exclude>r/src/main/scala/scala/Console.scala</exclude>
|
||||
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
|
||||
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
|
||||
<!--The following files are mechanical-->
|
||||
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
|
||||
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
|
||||
<executions>
|
||||
<execution>
|
||||
<id>verify.rat</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
|
|
@ -652,7 +531,7 @@
|
|||
<profile>
|
||||
<id>scala-2.10</id>
|
||||
<activation>
|
||||
<property><name>!scala-2.11</name></property>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<scala.version>2.10.5</scala.version>
|
||||
|
|
@ -662,9 +541,6 @@
|
|||
|
||||
<profile>
|
||||
<id>scala-2.11</id>
|
||||
<activation>
|
||||
<property><name>scala-2.11</name></property>
|
||||
</activation>
|
||||
<properties>
|
||||
<scala.version>2.11.7</scala.version>
|
||||
<scala.binary.version>2.11</scala.binary.version>
|
||||
|
|
@ -812,6 +688,143 @@
|
|||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>rat</id>
|
||||
<activation>
|
||||
<property><name>!skipRat</name></property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
<version>0.11</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/*.keywords</exclude>
|
||||
<exclude>reports/**</exclude>
|
||||
<exclude>**/.idea/</exclude>
|
||||
<exclude>**/*.iml</exclude>
|
||||
<exclude>.git/</exclude>
|
||||
<exclude>.github/*</exclude>
|
||||
<exclude>.gitignore</exclude>
|
||||
<exclude>.repository/</exclude>
|
||||
<exclude>.rat-excludes/</exclude>
|
||||
<exclude>.Rhistory</exclude>
|
||||
<exclude>**/*.diff</exclude>
|
||||
<exclude>**/*.patch</exclude>
|
||||
<exclude>**/*.avsc</exclude>
|
||||
<exclude>**/*.avro</exclude>
|
||||
<exclude>**/*.log</exclude>
|
||||
<exclude>**/test/resources/**</exclude>
|
||||
<exclude>**/.settings/*</exclude>
|
||||
<exclude>**/.classpath</exclude>
|
||||
<exclude>**/.project</exclude>
|
||||
<exclude>**/target/**</exclude>
|
||||
<exclude>**/derby.log</exclude>
|
||||
<exclude>**/metastore_db/</exclude>
|
||||
<exclude>**/logs/**</exclude>
|
||||
<exclude>**/run/**</exclude>
|
||||
<exclude>**/interpreter/**</exclude>
|
||||
<exclude>**/local-repo/**</exclude>
|
||||
<exclude>**/null/**</exclude>
|
||||
<exclude>**/notebook/**</exclude>
|
||||
<exclude>_tools/site/css/*</exclude>
|
||||
<exclude>**/README.md</exclude>
|
||||
<exclude>DEPENDENCIES</exclude>
|
||||
<exclude>DEPLOY.md</exclude>
|
||||
<exclude>CONTRIBUTING.md</exclude>
|
||||
<exclude>STYLE.md</exclude>
|
||||
<exclude>Roadmap.md</exclude>
|
||||
<exclude>**/licenses/**</exclude>
|
||||
<exclude>**/zeppelin-distribution/src/bin_license/**</exclude>
|
||||
<exclude>conf/interpreter.json</exclude>
|
||||
<exclude>conf/notebook-authorization.json</exclude>
|
||||
<exclude>conf/credentials.json</exclude>
|
||||
<exclude>conf/zeppelin-env.sh</exclude>
|
||||
<exclude>spark-*-bin*/**</exclude>
|
||||
<exclude>.spark-dist/**</exclude>
|
||||
<exclude>**/interpreter-setting.json</exclude>
|
||||
<exclude>**/constants.json</exclude>
|
||||
|
||||
<!-- bundled from bootstrap -->
|
||||
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
|
||||
<exclude>docs/assets/themes/zeppelin/css/style.css</exclude>
|
||||
<exclude>docs/assets/themes/zeppelin/js/docs.js</exclude>
|
||||
<exclude>docs/assets/themes/zeppelin/js/search.js</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/_jumbotron.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/_navigation.html</exclude>
|
||||
|
||||
<!-- bundled from jekyll-bootstrap -->
|
||||
<exclude>docs/404.html</exclude>
|
||||
<exclude>docs/_config.yml</exclude>
|
||||
<exclude>docs/_includes/JB/**</exclude>
|
||||
<exclude>docs/_layouts/**</exclude>
|
||||
<exclude>docs/_plugins/**</exclude>
|
||||
<exclude>docs/atom.xml</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/default.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/page.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/post.html</exclude>
|
||||
<exclude>docs/_includes/themes/zeppelin/settings.yml</exclude>
|
||||
<exclude>docs/Rakefile</exclude>
|
||||
<exclude>docs/rss.xml</exclude>
|
||||
<exclude>docs/sitemap.txt</exclude>
|
||||
<exclude>docs/search_data.json</exclude>
|
||||
<exclude>**/dependency-reduced-pom.xml</exclude>
|
||||
<exclude>docs/CONTRIBUTING.md</exclude>
|
||||
|
||||
<!-- bundled from anchor -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
|
||||
|
||||
<!-- bundled from toc -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/toc.js</exclude>
|
||||
|
||||
<!-- bundled from lunrjs -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/lunr.min.js</exclude>
|
||||
|
||||
<!-- bundled from jekyll -->
|
||||
<exclude>docs/assets/themes/zeppelin/css/syntax.css</exclude>
|
||||
|
||||
<!-- docs (website) build target dir -->
|
||||
<exclude>docs/_site/**</exclude>
|
||||
<exclude>docs/Gemfile.lock</exclude>
|
||||
|
||||
<exclude>**/horizontalbar_mockdata.txt</exclude>
|
||||
|
||||
<!-- compiled R packages (binaries) -->
|
||||
<exclude>R/lib/**</exclude>
|
||||
<exclude>r/lib/**</exclude>
|
||||
|
||||
<!--R-related files with alternative licenses-->
|
||||
<exclude>r/R/rzeppelin/R/globals.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/common.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
|
||||
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
|
||||
<exclude>r/src/main/scala/scala/Console.scala</exclude>
|
||||
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
|
||||
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
|
||||
<!--The following files are mechanical-->
|
||||
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
|
||||
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
|
||||
<executions>
|
||||
<execution>
|
||||
<id>verify.rat</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -134,10 +134,11 @@ public class PythonInterpreter extends Interpreter {
|
|||
|
||||
InterpreterResult result;
|
||||
if (pythonErrorIn(output)) {
|
||||
result = new InterpreterResult(Code.ERROR, output.replaceAll(">>>", "").trim());
|
||||
result = new InterpreterResult(Code.ERROR, output);
|
||||
} else {
|
||||
result = new InterpreterResult(Code.SUCCESS, output.replaceAll(">>>", "")
|
||||
.replaceAll("\\.\\.\\.", "").trim());
|
||||
// TODO(zjffdu), we should not do string replacement operation in the result, as it is
|
||||
// possible that the output contains the kind of pattern itself, e.g. print("...")
|
||||
result = new InterpreterResult(Code.SUCCESS, output.replaceAll("\\.\\.\\.", ""));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -149,8 +150,17 @@ public class PythonInterpreter extends Interpreter {
|
|||
* @return true if syntax error or exception has happened
|
||||
*/
|
||||
private boolean pythonErrorIn(String output) {
|
||||
Matcher errorMatcher = errorInLastLine.matcher(output);
|
||||
return errorMatcher.find();
|
||||
boolean isError = false;
|
||||
String[] outputMultiline = output.split("\n");
|
||||
Matcher errorMatcher;
|
||||
for (String row : outputMultiline) {
|
||||
errorMatcher = errorInLastLine.matcher(row);
|
||||
if (errorMatcher.find() == true) {
|
||||
isError = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return isError;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -265,4 +275,5 @@ public class PythonInterpreter extends Interpreter {
|
|||
public int getMaxResult() {
|
||||
return maxResult;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,12 +21,11 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.OutputStream;
|
||||
import java.lang.reflect.Field;
|
||||
|
||||
/**
|
||||
|
|
@ -34,11 +33,11 @@ import java.lang.reflect.Field;
|
|||
* Python process (REPL) used by python interpreter
|
||||
*/
|
||||
public class PythonProcess {
|
||||
Logger logger = LoggerFactory.getLogger(PythonProcess.class);
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PythonProcess.class);
|
||||
private static final String STATEMENT_END = "*!?flush reader!?*";
|
||||
InputStream stdout;
|
||||
OutputStream stdin;
|
||||
BufferedWriter writer;
|
||||
PrintWriter writer;
|
||||
BufferedReader reader;
|
||||
Process process;
|
||||
|
||||
|
|
@ -56,7 +55,7 @@ public class PythonProcess {
|
|||
process = builder.start();
|
||||
stdout = process.getInputStream();
|
||||
stdin = process.getOutputStream();
|
||||
writer = new BufferedWriter(new OutputStreamWriter(stdin));
|
||||
writer = new PrintWriter(stdin, true);
|
||||
reader = new BufferedReader(new InputStreamReader(stdout));
|
||||
try {
|
||||
pid = findPid();
|
||||
|
|
@ -85,22 +84,21 @@ public class PythonProcess {
|
|||
}
|
||||
|
||||
public String sendAndGetResult(String cmd) throws IOException {
|
||||
writer.write(cmd + "\n\n");
|
||||
writer.write("print (\"*!?flush reader!?*\")\n\n");
|
||||
writer.flush();
|
||||
|
||||
String output = "";
|
||||
String line;
|
||||
while (!(line = reader.readLine()).contains("*!?flush reader!?*")) {
|
||||
writer.println(cmd);
|
||||
writer.println();
|
||||
writer.println("\"" + STATEMENT_END + "\"");
|
||||
StringBuilder output = new StringBuilder();
|
||||
String line = null;
|
||||
while (!(line = reader.readLine()).contains(STATEMENT_END)) {
|
||||
logger.debug("Read line from python shell : " + line);
|
||||
if (line.equals("...")) {
|
||||
logger.warn("Syntax error ! ");
|
||||
output += "Syntax error ! ";
|
||||
output.append("Syntax error ! ");
|
||||
break;
|
||||
}
|
||||
output += "\r" + line + "\n";
|
||||
output.append(line + "\n");
|
||||
}
|
||||
return output;
|
||||
return output.toString();
|
||||
}
|
||||
|
||||
private long findPid() throws NoSuchFieldException, IllegalAccessException {
|
||||
|
|
|
|||
|
|
@ -19,90 +19,93 @@
|
|||
# Remove interactive mode displayhook
|
||||
import sys
|
||||
import signal
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
try:
|
||||
import StringIO as io
|
||||
from StringIO import StringIO
|
||||
except ImportError:
|
||||
import io as io
|
||||
|
||||
sys.displayhook = lambda x: None
|
||||
from io import StringIO
|
||||
|
||||
def intHandler(signum, frame): # Set the signal handler
|
||||
print ("Paragraph interrupted")
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
signal.signal(signal.SIGINT, intHandler)
|
||||
|
||||
# set prompt as empty string so that java side don't need to remove the prompt.
|
||||
sys.ps1=""
|
||||
|
||||
def help():
|
||||
print ('%html')
|
||||
print ('<h2>Python Interpreter help</h2>')
|
||||
print ('<h3>Python 2 & 3 compatibility</h3>')
|
||||
print ('<p>The interpreter is compatible with Python 2 & 3.<br/>')
|
||||
print ('To change Python version, ')
|
||||
print ('change in the interpreter configuration the python to the ')
|
||||
print ('desired version (example : python=/usr/bin/python3)</p>')
|
||||
print ('<h3>Python modules</h3>')
|
||||
print ('<p>The interpreter can use all modules already installed ')
|
||||
print ('(with pip, easy_install, etc)</p>')
|
||||
print ('<h3>Forms</h3>')
|
||||
print ('You must install py4j in order to use '
|
||||
'the form feature (pip install py4j)')
|
||||
print ('<h4>Input form</h4>')
|
||||
print ('<pre>print (z.input("f1","defaultValue"))</pre>')
|
||||
print ('<h4>Selection form</h4>')
|
||||
print ('<pre>print(z.select("f2", [("o1","1"), ("o2","2")],2))</pre>')
|
||||
print ('<h4>Checkbox form</h4>')
|
||||
print ('<pre> print("".join(z.checkbox("f3", [("o1","1"), '
|
||||
'("o2","2")],["1"])))</pre>')
|
||||
print ('<h3>Matplotlib graph</h3>')
|
||||
print ('<div>The interpreter can display matplotlib graph with ')
|
||||
print ('the function z.show()</div>')
|
||||
print ('<div> You need to already have matplotlib module installed ')
|
||||
print ('to use this functionality !</div><br/>')
|
||||
print ('''<pre>import matplotlib.pyplot as plt
|
||||
plt.figure()
|
||||
(.. ..)
|
||||
z.show(plt)
|
||||
plt.close()
|
||||
</pre>''')
|
||||
print ('<div><br/> z.show function can take optional parameters ')
|
||||
print ('to adapt graph width and height</div>')
|
||||
print ("<div><b>example </b>:")
|
||||
print ('''<pre>z.show(plt,width='50px')
|
||||
z.show(plt,height='150px') </pre></div>''')
|
||||
print ('<h3>Pandas DataFrame</h3>')
|
||||
print ('<div> You need to have Pandas module installed ')
|
||||
print ('to use this functionality (pip install pandas) !</div><br/>')
|
||||
print ("""
|
||||
<div>The interpreter can visualize Pandas DataFrame
|
||||
with the function z.show()
|
||||
<pre>
|
||||
import pandas as pd
|
||||
df = pd.read_csv("bank.csv", sep=";")
|
||||
z.show(df)
|
||||
</pre></div>
|
||||
""")
|
||||
print ('<h3>SQL over Pandas DataFrame</h3>')
|
||||
print ('<div> You need to have Pandas&Pandasql modules installed ')
|
||||
print ('to use this functionality (pip install pandas pandasql) !</div><br/>')
|
||||
print ("""
|
||||
<div>Python interpreter group includes %sql interpreter that can query
|
||||
Pandas DataFrames using SQL and visualize results using Zeppelin Table Display System
|
||||
print("""%html
|
||||
<h2>Python Interpreter help</h2>
|
||||
|
||||
<pre>
|
||||
%python
|
||||
import pandas as pd
|
||||
df = pd.read_csv("bank.csv", sep=";")
|
||||
</pre>
|
||||
<br />
|
||||
<h3>Python 2 & 3 compatibility</h3>
|
||||
<p>The interpreter is compatible with Python 2 & 3.<br/>
|
||||
To change Python version,
|
||||
change in the interpreter configuration the python to the
|
||||
desired version (example : python=/usr/bin/python3)</p>
|
||||
|
||||
<pre>
|
||||
%python.sql
|
||||
%sql
|
||||
SELECT * from df LIMIT 5
|
||||
</pre></div>
|
||||
""")
|
||||
<h3>Python modules</h3>
|
||||
<p>The interpreter can use all modules already installed
|
||||
(with pip, easy_install, etc)</p>
|
||||
|
||||
<h3>Forms</h3>
|
||||
You must install py4j in order to use
|
||||
the form feature (pip install py4j)
|
||||
<h4>Input form</h4>
|
||||
<pre>print (z.input("f1","defaultValue"))</pre>
|
||||
<h4>Selection form</h4>
|
||||
<pre>print(z.select("f2", [("o1","1"), ("o2","2")],2))</pre>
|
||||
<h4>Checkbox form</h4>
|
||||
<pre> print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))</pre>')
|
||||
|
||||
<h3>Matplotlib graph</h3>
|
||||
<div>The interpreter can display matplotlib graph with
|
||||
the function z.show()</div>
|
||||
<div> You need to already have matplotlib module installed
|
||||
to use this functionality !</div><br/>
|
||||
<pre>import matplotlib.pyplot as plt
|
||||
plt.figure()
|
||||
(.. ..)
|
||||
z.show(plt)
|
||||
plt.close()
|
||||
</pre>
|
||||
<div><br/> z.show function can take optional parameters
|
||||
to adapt graph dimensions (width and height) and format (png or svg)</div>
|
||||
<div><b>example </b>:
|
||||
<pre>z.show(plt,width='50px
|
||||
z.show(plt,height='150px', fmt='svg') </pre></div>
|
||||
|
||||
<h3>Pandas DataFrame</h3>
|
||||
<div> You need to have Pandas module installed
|
||||
to use this functionality (pip install pandas) !</div><br/>
|
||||
<div>The interpreter can visualize Pandas DataFrame
|
||||
with the function z.show()
|
||||
<pre>
|
||||
import pandas as pd
|
||||
df = pd.read_csv("bank.csv", sep=";")
|
||||
z.show(df)
|
||||
</pre></div>
|
||||
|
||||
<h3>SQL over Pandas DataFrame</h3>
|
||||
<div> You need to have Pandas&Pandasql modules installed
|
||||
to use this functionality (pip install pandas pandasql) !</div><br/>
|
||||
|
||||
<div>Python interpreter group includes %sql interpreter that can query
|
||||
Pandas DataFrames using SQL and visualize results using Zeppelin Table Display System
|
||||
|
||||
<pre>
|
||||
%python
|
||||
import pandas as pd
|
||||
df = pd.read_csv("bank.csv", sep=";")
|
||||
</pre>
|
||||
<br />
|
||||
<pre>
|
||||
%python.sql
|
||||
%sql
|
||||
SELECT * from df LIMIT 5
|
||||
</pre>
|
||||
</div>
|
||||
""")
|
||||
|
||||
|
||||
class PyZeppelinContext(object):
|
||||
|
|
@ -112,18 +115,17 @@ class PyZeppelinContext(object):
|
|||
errorMsg = "You must install py4j Python module " \
|
||||
"(pip install py4j) to use Zeppelin dynamic forms features"
|
||||
|
||||
def __init__(self, zc):
|
||||
self.z = zc
|
||||
def __init__(self):
|
||||
self.max_result = 1000
|
||||
|
||||
def input(self, name, defaultValue=""):
|
||||
print (self.errorMsg)
|
||||
print(self.errorMsg)
|
||||
|
||||
def select(self, name, options, defaultValue=""):
|
||||
print (self.errorMsg)
|
||||
print(self.errorMsg)
|
||||
|
||||
def checkbox(self, name, options, defaultChecked=[]):
|
||||
print (self.errorMsg)
|
||||
print(self.errorMsg)
|
||||
|
||||
def show(self, p, **kwargs):
|
||||
if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
|
||||
|
|
@ -139,20 +141,20 @@ class PyZeppelinContext(object):
|
|||
"""Pretty prints DF using Table Display System
|
||||
"""
|
||||
limit = len(df) > self.max_result
|
||||
header_buf = io.StringIO("")
|
||||
header_buf.write(df.columns[0])
|
||||
header_buf = StringIO("")
|
||||
header_buf.write(str(df.columns[0]))
|
||||
for col in df.columns[1:]:
|
||||
header_buf.write("\t")
|
||||
header_buf.write(col)
|
||||
header_buf.write(str(col))
|
||||
header_buf.write("\n")
|
||||
|
||||
body_buf = io.StringIO("")
|
||||
body_buf = StringIO("")
|
||||
rows = df.head(self.max_result).values if limit else df.values
|
||||
for row in rows:
|
||||
body_buf.write(row[0])
|
||||
body_buf.write(str(row[0]))
|
||||
for cell in row[1:]:
|
||||
body_buf.write("\t")
|
||||
body_buf.write(cell)
|
||||
body_buf.write(str(cell))
|
||||
body_buf.write("\n")
|
||||
body_buf.seek(0); header_buf.seek(0)
|
||||
#TODO(bzz): fix it, so it shows red notice, as in Spark
|
||||
|
|
@ -162,21 +164,29 @@ class PyZeppelinContext(object):
|
|||
#)
|
||||
body_buf.close(); header_buf.close()
|
||||
|
||||
def show_matplotlib(self, p, width="0", height="0", **kwargs):
|
||||
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
|
||||
**kwargs):
|
||||
"""Matplotlib show function
|
||||
"""
|
||||
img = io.StringIO()
|
||||
p.savefig(img, format='svg')
|
||||
img.seek(0)
|
||||
style = ""
|
||||
if (width != "0"):
|
||||
style += 'width:' + width
|
||||
if (height != "0"):
|
||||
if (len(style) != 0):
|
||||
style += ","
|
||||
style += 'height:' + height
|
||||
print("%html <div style='" + style + "'>" + img.read() + "<div>")
|
||||
if fmt == "png":
|
||||
img = BytesIO()
|
||||
p.savefig(img, format=fmt)
|
||||
img_str = b"data:image/png;base64,"
|
||||
img_str += base64.b64encode(img.getvalue().strip())
|
||||
img_tag = "<img src={img} style='width={width};height:{height}'>"
|
||||
# Decoding is necessary for Python 3 compability
|
||||
img_str = img_str.decode("ascii")
|
||||
img_str = img_tag.format(img=img_str, width=width, height=height)
|
||||
elif fmt == "svg":
|
||||
img = StringIO()
|
||||
p.savefig(img, format=fmt)
|
||||
img_str = img.getvalue()
|
||||
else:
|
||||
raise ValueError("fmt must be 'png' or 'svg'")
|
||||
|
||||
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
|
||||
print(html.format(width=width, height=height, img=img_str))
|
||||
img.close()
|
||||
|
||||
|
||||
z = PyZeppelinContext("")
|
||||
z = PyZeppelinContext()
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
|
|||
class Py4jZeppelinContext(PyZeppelinContext):
|
||||
"""A context impl that uses Py4j to communicate to JVM
|
||||
"""
|
||||
def __init__(self, zc):
|
||||
super(Py4jZeppelinContext, self).__init__(zc)
|
||||
def __init__(self, z):
|
||||
self.z = z
|
||||
self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
|
||||
self.javaList = gateway.jvm.java.util.ArrayList
|
||||
self.max_result = 1000 #TODO(bzz): read `zeppelin.python.maxResult` from JVM
|
||||
self.max_result = self.z.getMaxResult()
|
||||
|
||||
def input(self, name, defaultValue=""):
|
||||
return self.z.getGui().input(name, defaultValue)
|
||||
|
|
|
|||
|
|
@ -153,4 +153,24 @@ public class PythonInterpreterPandasSqlTest {
|
|||
assertTrue(ret.message().length() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void showDataFrame() {
|
||||
InterpreterResult ret;
|
||||
ret = python.interpret("import pandas as pd", context);
|
||||
ret = python.interpret("import numpy as np", context);
|
||||
|
||||
// given a Pandas DataFrame with non-text data
|
||||
ret = python.interpret("d1 = {1 : [np.nan, 1, 2, 3], 'two' : [3., 4., 5., 6.7]}", context);
|
||||
ret = python.interpret("df1 = pd.DataFrame(d1)", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
|
||||
// when
|
||||
ret = python.interpret("z.show(df1)", context);
|
||||
|
||||
// then
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertEquals(ret.message(), Type.TABLE, ret.type());
|
||||
assertTrue(ret.message().indexOf("nan") > 0);
|
||||
assertTrue(ret.message().indexOf("6.7") > 0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ public class PythonInterpreterTest {
|
|||
*/
|
||||
@Test
|
||||
public void testPy4jInstalled() throws IOException, InterruptedException {
|
||||
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(">>>");
|
||||
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
|
||||
|
||||
pythonInterpreter.open();
|
||||
Integer py4jPort = pythonInterpreter.getPy4jPort();
|
||||
|
|
@ -137,7 +137,7 @@ public class PythonInterpreterTest {
|
|||
@Test
|
||||
public void testClose() throws IOException, InterruptedException {
|
||||
//given: py4j is installed
|
||||
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(">>>");
|
||||
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
|
||||
|
||||
pythonInterpreter.open();
|
||||
Integer py4jPort = pythonInterpreter.getPy4jPort();
|
||||
|
|
@ -210,12 +210,33 @@ public class PythonInterpreterTest {
|
|||
String output = "";
|
||||
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
output += ">>>" + lines[i];
|
||||
output += lines[i];
|
||||
}
|
||||
return output;
|
||||
} else {
|
||||
return ">>>";
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void checkMultiRowErrorFails() {
|
||||
PythonInterpreter pythonInterpreter = new PythonInterpreter(
|
||||
PythonInterpreterTest.getPythonTestProperties()
|
||||
);
|
||||
pythonInterpreter.open();
|
||||
String codeRaiseException = "raise Exception(\"test exception\")";
|
||||
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
|
||||
|
||||
assertNotNull("Interpreter result for raise exception is Null", ret);
|
||||
|
||||
assertEquals(InterpreterResult.Code.ERROR, ret.code());
|
||||
assertTrue(ret.message().length() > 0);
|
||||
|
||||
assertNotNull("Interpreter result for text is Null", ret);
|
||||
String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
|
||||
ret = pythonInterpreter.interpret(codePrintText, null);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertTrue(ret.message().length() > 0);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue