Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-1023

Browse source 
rebase
This commit is contained in:
astroshim 2016-06-27 22:38:47 +09:00
commit 1b94ae12b0
243 changed files with 5693 additions and 3141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored
View file

@ -32,7 +32,7 @@ conf/credentials.json
spark/dependency-reduced-pom.xml
reports
#webapp
# webapp
zeppelin-web/node_modules
zeppelin-web/dist
zeppelin-web/.tmp
@ -41,8 +41,9 @@ zeppelin-web/bower_components
**nbproject/
**node/
#R
# R
/r/lib/
.Rhistory
# project level
/logs/
@ -80,7 +81,7 @@ Thumbs.db
.project
.settings/
#intelliJ IDEA project files
# intelliJ IDEA project files
.idea/
*.iml

3
.travis.yml
View file

@ -67,8 +67,7 @@ before_install:
- echo 'R_LIBS=~/R' > ~/.Renviron
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
- export R_LIBS='~/R'
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
install:
- mvn $BUILD_FLAG $PROFILE -B

8
LICENSE
View file

@ -235,7 +235,13 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) jekyll-bootstrap 0.3.0 (https://github.com/plusjade/jekyll-bootstrap) - https://github.com/plusjade/jekyll-bootstrap
(The MIT License) jekyll 1.3.0 (http://jekyllrb.com/) - https://github.com/jekyll/jekyll/blob/v1.3.0/LICENSE
========================================================================
MIT-style licenses
========================================================================
The following components are provided under the MIT-style license. See project link for details.
The text of each license is also included at licenses/LICENSE-[project]-[version].txt.
(MIT Style) jekyll-table-of-contents (https://github.com/ghiculescu/jekyll-table-of-contents) - https://github.com/ghiculescu/jekyll-table-of-contents/blob/master/LICENSE.txt
========================================================================
Apache licenses

9
alluxio/pom.xml
View file

@ -132,15 +132,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

5
alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java
View file

@ -28,6 +28,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -174,13 +175,13 @@ public class AlluxioInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
String lastWord = "";
if (words.length > 0) {
lastWord = words[ words.length - 1 ];
}
ArrayList<String> voices = new ArrayList<String>();
ArrayList voices = new ArrayList<>();
for (String command : keywords) {
if (command.startsWith(lastWord)) {
voices.add(command);

19
alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java
View file

@ -31,6 +31,7 @@ import alluxio.client.WriteType;
import alluxio.client.file.URIStatus;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.*;
import alluxio.Constants;
@ -76,19 +77,19 @@ public class AlluxioInterpreterTest {
@Test
public void testCompletion() {
List<String> expectedResultOne = Arrays.asList("cat", "chgrp",
List expectedResultOne = Arrays.asList("cat", "chgrp",
"chmod", "chown", "copyFromLocal", "copyToLocal", "count",
"createLineage");
List<String> expectedResultTwo = Arrays.asList("copyFromLocal",
List expectedResultTwo = Arrays.asList("copyFromLocal",
"copyToLocal", "count");
List<String> expectedResultThree = Arrays.asList("copyFromLocal", "copyToLocal");
List<String> expectedResultNone = new ArrayList<String>();
List expectedResultThree = Arrays.asList("copyFromLocal", "copyToLocal");
List expectedResultNone = new ArrayList<String>();
List<String> resultOne = alluxioInterpreter.completion("c", 0);
List<String> resultTwo = alluxioInterpreter.completion("co", 0);
List<String> resultThree = alluxioInterpreter.completion("copy", 0);
List<String> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
List<String> resultAll = alluxioInterpreter.completion("", 0);
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);
List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0);
List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0);
Assert.assertEquals(expectedResultOne, resultOne);
Assert.assertEquals(expectedResultTwo, resultTwo);

9
angular/pom.xml
View file

@ -60,15 +60,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

5
angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java
View file

@ -26,6 +26,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -69,8 +70,8 @@ public class AngularInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
return new LinkedList<String>();
public List<InterpreterCompletion> completion(String buf, int cursor) {
return new LinkedList<>();
}
@Override

45
bin/install-interpreter.sh Executable file
View file

@ -0,0 +1,45 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Run Zeppelin
#
bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
. "${bin}/common.sh"
ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log"
JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
addJarInDir "${ZEPPELIN_HOME}/lib"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@}

9
cassandra/pom.xml
View file

@ -230,15 +230,6 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

5
cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java
View file

@ -23,6 +23,7 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -139,7 +140,7 @@ public class CassandraInterpreter extends Interpreter {
public static final String LOGGING_DOWNGRADING_RETRY = "LOGGING_DOWNGRADING";
public static final String LOGGING_FALLTHROUGH_RETRY = "LOGGING_FALLTHROUGH";
public static final List<String> NO_COMPLETION = new ArrayList<>();
public static final List NO_COMPLETION = new ArrayList<>();
InterpreterLogic helper;
Cluster cluster;
@ -320,7 +321,7 @@ public class CassandraInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return NO_COMPLETION;
}

35
conf/interpreter-list Normal file
View file

@ -0,0 +1,35 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# [name] [maven artifact] [description]
alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter
angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering
cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter
elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter
file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter
flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter
hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter
ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter
jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter
kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter
lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter
livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter
md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support
postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter
python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter
shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command

31
conf/shiro.ini
View file

@ -25,19 +25,44 @@ user3 = password4, role2
# Sample LDAP configuration, for user Authentication, currently tested for single Realm
[main]
#ldapRealm = org.apache.shiro.realm.ldap.JndiLdapRealm
#ldapRealm.userDnTemplate = cn={0},cn=engg,ou=testdomain,dc=testdomain,dc=com
#ldapRealm.contextFactory.url = ldap://ldaphost:389
### A sample for configuring Active Directory Realm
#activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
#activeDirectoryRealm.systemUsername = userNameA
#activeDirectoryRealm.systemPassword = passwordA
#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
#activeDirectoryRealm.url = ldap://ldap.test.com:389
#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
#activeDirectoryRealm.authorizationCachingEnabled = false
### A sample for configuring LDAP Directory Realm
#ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
## search base for ldap groups (only relevant for LdapGroupRealm):
#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
#ldapRealm.contextFactory.url = ldap://ldap.test.com:389
#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
### If caching of user is required then uncomment below lines
#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
#securityManager.cacheManager = $cacheManager
securityManager.sessionManager = $sessionManager
# 86,400,000 milliseconds = 24 hour
securityManager.sessionManager.globalSessionTimeout = 86400000
shiro.loginUrl = /api/login
[roles]
role1 = *
role2 = *
role3 = *
[urls]
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# authc means Form based Auth Security
# To enfore security, comment the line below and uncomment the next one
/api/version = anon
/** = anon

3
conf/zeppelin-env.cmd.template
View file

@ -55,12 +55,13 @@ REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration
REM Pyspark (supported with Spark 1.2.1 and above)
REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers.
REM set PYTHONPATH
REM set PYTHONPATH
REM Spark interpreter options
REM
REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default.
REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default.
REM set ZEPPELIN_SPARK_IMPORTIMPLICIT REM Import implicits, UDF collection, and sql if set true. true by default.
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default.
REM ZeppelinHub connection configuration

5
conf/zeppelin-env.sh.template
View file

@ -55,15 +55,17 @@
# Pyspark (supported with Spark 1.2.1 and above)
# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
# export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers.
# export PYTHONPATH
# export PYTHONPATH
## Spark interpreter options ##
##
# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default.
# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default.
# export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default.
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
#### HBase interpreter configuration ####
## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
@ -75,4 +77,3 @@
# export ZEPPELINHUB_API_ADDRESS # Refers to the address of the ZeppelinHub service in use
# export ZEPPELINHUB_API_TOKEN # Refers to the Zeppelin instance token of the user
# export ZEPPELINHUB_USER_KEY # Optional, when using Zeppelin with authentication.

2
conf/zeppelin-site.xml.template
View file

@ -178,7 +178,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

61
dev/common_release.sh Normal file
View file

@ -0,0 +1,61 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# common fucntions
if [[ -z "${TAR}" ]]; then
TAR="/usr/bin/tar"
fi
if [[ -z "${SHASUM}" ]]; then
SHASUM="/usr/bin/shasum"
fi
if [[ -z "${WORKING_DIR}" ]]; then
WORKING_DIR="/tmp/zeppelin-release"
fi
mkdir "${WORKING_DIR}"
usage() {
echo "usage) $0 [Release version] [Branch or Tag]"
echo " ex. $0 0.6.0 v0.6.0"
exit 1
}
function git_clone() {
echo "Clone the source"
# clone source
git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git "${WORKING_DIR}/zeppelin"
if [[ $? -ne 0 ]]; then
echo "Can not clone source repository"
exit 1
fi
cd "${WORKING_DIR}/zeppelin"
git checkout "${GIT_TAG}"
echo "Checked out ${GIT_TAG}"
# remove unnecessary files
rm "${WORKING_DIR}/zeppelin/.gitignore"
rm -rf "${WORKING_DIR}/zeppelin/.git"
rm -rf "${WORKING_DIR}/zeppelin/.github"
rm -rf "${WORKING_DIR}/zeppelin/docs"
}

166
dev/create_release.sh
View file

@ -18,114 +18,94 @@
#
# The script helps making a release.
# You need specify a release name and branch|tag name.
# You need to specify release version and branch|tag name.
#
# Here's some helpful documents for the release
# Here are some helpful documents for the release.
# http://www.apache.org/dev/release.html
# http://www.apache.org/dev/release-publishing
# http://www.apache.org/dev/release-signing.html
# http://www.apache.org/dev/publishing-maven-artifacts.html
if [[ -z "${TAR}" ]]; then
TAR=/usr/bin/tar
fi
BASEDIR="$(dirname "$0")"
. "${BASEDIR}/common_release.sh"
echo "${BASEDIR}/common_release.sh"
if [[ -z "${SHASUM}" ]]; then
SHASUM="/usr/bin/shasum -a 512"
fi
if [[ -z "${WORKING_DIR}" ]]; then
WORKING_DIR=/tmp/zeppelin-release
if [[ $# -ne 2 ]]; then
usage
fi
if [[ -z "${GPG_PASSPHRASE}" ]]; then
echo "You need GPG_PASSPHRASE variable set"
exit 1
echo "You need GPG_PASSPHRASE variable set"
exit 1
fi
RELEASE_VERSION="$1"
GIT_TAG="$2"
if [[ $# -ne 2 ]]; then
echo "usage) $0 [Release name] [Branch or Tag]"
echo " ex. $0 0.6.0 branch-0.6"
exit 1
fi
function make_source_package() {
# create source package
cd ${WORKING_DIR}
cp -r "zeppelin" "zeppelin-${RELEASE_VERSION}"
${TAR} cvzf "zeppelin-${RELEASE_VERSION}.tgz" "zeppelin-${RELEASE_VERSION}"
RELEASE_NAME="${1}"
BRANCH="${2}"
if [[ -d "${WORKING_DIR}" ]]; then
echo "Dir ${WORKING_DIR} already exists"
exit 1
fi
mkdir ${WORKING_DIR}
echo "Cloning the source and packaging"
# clone source
git clone -b ${BRANCH} git@github.com:apache/zeppelin.git ${WORKING_DIR}/zeppelin
if [[ $? -ne 0 ]]; then
echo "Can not clone source repository"
exit 1
fi
# remove unnecessary files
rm ${WORKING_DIR}/zeppelin/.gitignore
rm -rf ${WORKING_DIR}/zeppelin/.git
# create source package
cd ${WORKING_DIR}
cp -r zeppelin zeppelin-${RELEASE_NAME}
${TAR} cvzf zeppelin-${RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME}
echo "Signing the source package"
cd ${WORKING_DIR}
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}.tgz.asc --detach-sig ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.md5
${SHASUM} zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.sha512
function make_binary_release() {
BIN_RELEASE_NAME="${1}"
BUILD_FLAGS="${2}"
cp -r ${WORKING_DIR}/zeppelin ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
cd ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}"
mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}
if [[ $? -ne 0 ]]; then
echo "Build failed. ${BUILD_FLAGS}"
exit 1
fi
# re-create package with proper dir name with binary license
cd zeppelin-distribution/target/zeppelin-*
mv zeppelin-* zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
cat ../../src/bin_license/LICENSE >> zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/LICENSE
cat ../../src/bin_license/NOTICE >> zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/NOTICE
cp ../../src/bin_license/licenses/* zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/licenses/
${TAR} cvzf zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
# sign bin package
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc --detach-sig zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5
${SHASUM} zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5 ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512 ${WORKING_DIR}/
# clean up build dir
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
echo "Signing the source package"
cd "${WORKING_DIR}"
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \
--output "zeppelin-${RELEASE_VERSION}.tgz.asc" \
--detach-sig "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz"
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 \
--print-md MD5 "zeppelin-${RELEASE_VERSION}.tgz" > \
"${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.md5"
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 \
--print-md SHA512 "zeppelin-${RELEASE_VERSION}.tgz" > \
"${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.sha512"
}
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark"
function make_binary_release() {
BIN_RELEASE_NAME="$1"
BUILD_FLAGS="$2"
cp -r "${WORKING_DIR}/zeppelin" "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
cd "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}"
mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}
if [[ $? -ne 0 ]]; then
echo "Build failed. ${BUILD_FLAGS}"
exit 1
fi
# re-create package with proper dir name with binary license
cd zeppelin-distribution/target/zeppelin-*
mv zeppelin-* "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
cat ../../src/bin_license/LICENSE >> "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/LICENSE"
cat ../../src/bin_license/NOTICE >> "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/NOTICE"
cp ../../src/bin_license/licenses/* "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/licenses/"
${TAR} cvzf "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
# sign bin package
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \
--output "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc" \
--detach-sig "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz"
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --print-md MD5 \
"zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" > \
"zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5"
${SHASUM} -a 512 "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" > \
"zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512"
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" "${WORKING_DIR}/"
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc" "${WORKING_DIR}/"
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5" "${WORKING_DIR}/"
mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512" "${WORKING_DIR}/"
# clean up build dir
rm -rf "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}"
}
git_clone
make_source_package
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell"
# remove non release files and dirs
rm -rf ${WORKING_DIR}/zeppelin
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}
rm -rf "${WORKING_DIR}/zeppelin"
rm -rf "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}"
echo "Release files are created under ${WORKING_DIR}"

138
dev/publish_release.sh Executable file
View file

@ -0,0 +1,138 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The script helps publishing release to maven.
# You need to specify release version and branch|tag name.
#
# Here's some helpful documents for the release.
# http://www.apache.org/dev/publishing-maven-artifacts.html
BASEDIR="$(dirname "$0")"
. "${BASEDIR}/common_release.sh"
if [[ $# -ne 2 ]]; then
usage
fi
for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do
if [[ -z "${!var}" ]]; then
echo "You need ${var} variable set"
exit 1
fi
done
export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m"
RED='\033[0;31m'
NC='\033[0m' # No Color
RELEASE_VERSION="$1"
GIT_TAG="$2"
PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"
function cleanup() {
echo "Remove working directory and maven local repository"
rm -rf ${WORKING_DIR}
rm -rf ${tmp_repo}
}
function curl_error() {
ret=${1}
if [[ $ret -ne 0 ]]; then
echo "curl response code is: ($ret)"
echo "See https://curl.haxx.se/libcurl/c/libcurl-errors.html to know the detailed cause of error."
echo -e "${RED}Failed to publish maven artifact to staging repository."
echo -e "IMPORTANT: You will have to re-run publish_release.sh to complete maven artifact publish.${NC}"
cleanup
exit 1
fi
}
function publish_to_maven() {
cd "${WORKING_DIR}/zeppelin"
# Force release version
mvn versions:set -DnewVersion="${RELEASE_VERSION}"
# Using Nexus API documented here:
# https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API
echo "Creating Nexus staging repository"
repo_request="<promoteRequest><data><description>Apache Zeppelin ${RELEASE_VERSION}</description></data></promoteRequest>"
out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \
-H 'Content-Type:application/xml' -v \
"${NEXUS_STAGING}/profiles/${NEXUS_PROFILE}/start")"
create_ret=$?
curl_error $create_ret
staged_repo_id="$(echo "${out}" | sed -e 's/.*\(orgapachezeppelin-[0-9]\{4\}\).*/\1/')"
echo "Created Nexus staging repository: ${staged_repo_id}"
tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)"
echo "mvn clean install -Ppublish-distr \
-Dmaven.repo.local=${tmp_repo} \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}"
mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" \
${PUBLISH_PROFILES} ${PROJECT_OPTIONS}
if [[ $? -ne 0 ]]; then
echo "Build failed."
exit 1
fi
pushd "${tmp_repo}/org/apache/zeppelin"
find . -type f | grep -v '\.jar$' | grep -v '\.pom$' |grep -v '\.war$' | xargs rm
echo "Creating hash and signature files"
for file in $(find . -type f); do
echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --output "${file}.asc" \
--detach-sig --armor "${file}"
md5 -q "${file}" > "${file}.md5"
${SHASUM} -a 1 "${file}" | cut -f1 -d' ' > "${file}.sha1"
done
nexus_upload="${NEXUS_STAGING}/deployByRepositoryId/${staged_repo_id}"
echo "Uplading files to ${nexus_upload}"
for file in $(find . -type f); do
# strip leading ./
file_short="$(echo "${file}" | sed -e 's/\.\///')"
dest_url="${nexus_upload}/org/apache/zeppelin/$file_short"
echo " Uploading ${file_short}"
curl -u "${ASF_USERID}:${ASF_PASSWORD}" --upload-file "${file_short}" "${dest_url}"
upload_ret=$?
curl_error $upload_ret
done
echo "Closing nexus staging repository"
repo_request="<promoteRequest><data><stagedRepositoryId>${staged_repo_id}</stagedRepositoryId><description>Apache Zeppelin ${RELEASE_VERSION}</description></data></promoteRequest>"
out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \
-H 'Content-Type:application/xml' -v \
"${NEXUS_STAGING}}/profiles/${NEXUS_PROFILE}/finish")"
close_ret=$?
curl_error $close_ret
echo "Closed Nexus staging repository: ${staged_repo_id}"
popd
echo "Complete publishing maven artifacts to apache staging repository"
echo "Once release candidate pass the vote, do not forget to hit the release button in https://repository.apache.org"
}
git_clone
publish_to_maven
cleanup

6
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -21,8 +21,8 @@
<li><a href="{{BASE_PATH}}/index.html">What is Apache Zeppelin ?</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Getting Started</b><span></li>
<li><a href="{{BASE_PATH}}/install/install.html">Quick Start</a></li>
<li><a href="{{BASE_PATH}}/install/install.html#zeppelin-configuration">Configuration</a></li>
<li><a href="{{BASE_PATH}}/install/install.html">Install</a></li>
<li><a href="{{BASE_PATH}}/install/install.html#apache-zeppelin-configuration">Configuration</a></li>
<li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li>
<li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li>
<li role="separator" class="divider"></li>
@ -42,6 +42,7 @@
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Usage</b><span></li>
<li><a href="{{BASE_PATH}}/manual/interpreterinstallation.html">Interpreter Installation</a></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
<li role="separator" class="divider"></li>
@ -111,3 +112,4 @@
</nav><!--/.navbar-collapse -->
</div>
</div>

1
docs/_includes/themes/zeppelin/default.html
View file

@ -33,6 +33,7 @@
<script src="{{ ASSET_PATH }}/bootstrap/js/bootstrap.min.js"></script>
<script src="{{ ASSET_PATH }}/js/docs.js"></script>
<script src="{{ ASSET_PATH }}/js/anchor.min.js"></script>
<script src="{{ ASSET_PATH }}/js/toc.js"></script>
<!-- atom & rss feed -->
<link href="{{ BASE_PATH }}{{ site.JB.atom_path }}" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed">

16
docs/assets/themes/zeppelin/css/style.css
View file

@ -433,6 +433,8 @@ a.anchor {
word-break: keep-all;
-webkit-overflow-scrolling: touch;
font-size: 90%;
margin-top: 16px;
margin-bottom: 16px;
}
.content table th {
font-weight: bold;
@ -543,6 +545,20 @@ a.anchor {
a.anchorjs-link:hover { text-decoration: none; }
/* Table of Contents(TOC) */
#toc {
padding-top: 12px;
padding-bottom: 12px;
}
#toc ul {
margin-left: -14px;
}
#toc ul ul {
margin-left: -18px;
}
/* Custom, iPhone Retina */
@media only screen and (max-width: 480px) {
.jumbotron h1 {

0
docs/interpreter/screenshots/backtoscala.png → docs/assets/themes/zeppelin/img/docs-img/backtoscala.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

0
docs/interpreter/screenshots/knitgeo.png → docs/assets/themes/zeppelin/img/docs-img/knitgeo.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 58 KiB

After

Width:  |  Height:  |  Size: 58 KiB

0
docs/interpreter/screenshots/knitmotion.png → docs/assets/themes/zeppelin/img/docs-img/knitmotion.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 33 KiB

After

Width:  |  Height:  |  Size: 33 KiB

0
docs/interpreter/screenshots/knitstock.png → docs/assets/themes/zeppelin/img/docs-img/knitstock.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 106 KiB

0
docs/interpreter/screenshots/pythonMatplotlib.png → docs/assets/themes/zeppelin/img/docs-img/pythonMatplotlib.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

0
docs/interpreter/screenshots/repl2plus2.png → docs/assets/themes/zeppelin/img/docs-img/repl2plus2.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 13 KiB

0
docs/interpreter/screenshots/replhead.png → docs/assets/themes/zeppelin/img/docs-img/replhead.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

0
docs/interpreter/screenshots/replhist.png → docs/assets/themes/zeppelin/img/docs-img/replhist.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

0
docs/interpreter/screenshots/sparkrfaithful.png → docs/assets/themes/zeppelin/img/docs-img/sparkrfaithful.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 51 KiB

0
docs/interpreter/screenshots/varr1.png → docs/assets/themes/zeppelin/img/docs-img/varr1.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 16 KiB

After

Width:  |  Height:  |  Size: 16 KiB

0
docs/interpreter/screenshots/varr2.png → docs/assets/themes/zeppelin/img/docs-img/varr2.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

0
docs/interpreter/screenshots/varscala.png → docs/assets/themes/zeppelin/img/docs-img/varscala.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 21 KiB

4
docs/assets/themes/zeppelin/js/docs.js
View file

@ -118,6 +118,10 @@ $(function() {
maybeScrollToHash();
});
$(document).ready(function() {
$('#toc').toc();
});
// Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers
// will try to do *their* initial scroll after running the onReady handler.
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });

98
docs/assets/themes/zeppelin/js/toc.js Executable file
View file

@ -0,0 +1,98 @@
// https://github.com/ghiculescu/jekyll-table-of-contents
(function($){
$.fn.toc = function(options) {
var defaults = {
noBackToTopLinks: false,
title: '<i></i>',
minimumHeaders: 2,
headers: 'h2, h3',
listType: 'ul', // values: [ol|ul]
showEffect: 'none', // values: [show|slideDown|fadeIn|none]
showSpeed: '0', // set to 0 to deactivate effect
classes: { list: '',
item: ''
}
},
settings = $.extend(defaults, options);
function fixedEncodeURIComponent (str) {
return encodeURIComponent(str).replace(/[!'()*]/g, function(c) {
return '%' + c.charCodeAt(0).toString(16);
});
}
function createLink (header) {
var innerText = (header.textContent === undefined) ? header.innerText : header.textContent;
return "<a href='#" + fixedEncodeURIComponent(header.id) + "'>" + innerText + "</a>";
}
var headers = $(settings.headers).filter(function() {
// get all headers with an ID
var previousSiblingName = $(this).prev().attr( "name" );
if (!this.id && previousSiblingName) {
this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") );
}
return this.id;
}), output = $(this);
if (!headers.length || headers.length < settings.minimumHeaders || !output.length) {
$(this).hide();
return;
}
if (0 === settings.showSpeed) {
settings.showEffect = 'none';
}
var render = {
show: function() { output.hide().html(html).show(settings.showSpeed); },
slideDown: function() { output.hide().html(html).slideDown(settings.showSpeed); },
fadeIn: function() { output.hide().html(html).fadeIn(settings.showSpeed); },
none: function() { output.html(html); }
};
var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); };
var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0];
var return_to_top = '<i class="icon-arrow-up back-to-top"> </i>';
var level = get_level(headers[0]),
this_level,
html = settings.title + " <" +settings.listType + " class=\"" + settings.classes.list +"\">";
headers.on('click', function() {
if (!settings.noBackToTopLinks) {
window.location.hash = this.id;
}
})
.addClass('clickable-header')
.each(function(_, header) {
this_level = get_level(header);
if (!settings.noBackToTopLinks && this_level === highest_level) {
$(header).addClass('top-level-header').after(return_to_top);
}
if (this_level === level) // same level as before; same indenting
html += "<li class=\"" + settings.classes.item + "\">" + createLink(header);
else if (this_level <= level){ // higher level than before; end parent ol
for(i = this_level; i < level; i++) {
html += "</li></"+settings.listType+">"
}
html += "<li class=\"" + settings.classes.item + "\">" + createLink(header);
}
else if (this_level > level) { // lower level than before; expand the previous to contain a ol
for(i = this_level; i > level; i--) {
html += "<" + settings.listType + " class=\"" + settings.classes.list +"\">" +
"<li class=\"" + settings.classes.item + "\">"
}
html += createLink(header);
}
level = this_level; // update for the next one
});
html += "</"+settings.listType+">";
if (!settings.noBackToTopLinks) {
$(document).on('click', '.back-to-top', function() {
$(window).scrollTop(0);
window.location.hash = '';
});
}
render[settings.showEffect]();
};
})(jQuery);

25
docs/development/howtocontribute.md
View file

@ -7,8 +7,9 @@ group: development
# Contributing to Apache Zeppelin ( Code )
## IMPORTANT
Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
<div id="toc"></div>
> **NOTE :** Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
## Setting up
@ -22,7 +23,7 @@ Since Zeppelin uses Git for it's SCM system, you need git client installed in yo
You are free to use whatever IDE you prefer, or your favorite command line editor.
### Build Tools
#### Build Tools
To build the code, install
@ -46,10 +47,10 @@ You may also want to develop against a specific branch. For example, for branch-
git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin
```
#### Fork repository
If you want not only build Zeppelin but also make any changes, then you need fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request.
Apache Zeppelin follows [Fork & Pull](https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) as a source control workflow.
If you want to not only build Zeppelin but also make any changes, then you need to fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request.
###Build
### Build
```
mvn install
@ -67,6 +68,8 @@ To build with specific spark / hadoop version
mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x
```
For the further
### Run Zeppelin server in development mode
```
@ -88,21 +91,19 @@ Server will be run on [http://localhost:8080](http://localhost:8080).
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
To regenerate the code, install **thrift-0.9.0** and change directory into Zeppelin source directory. and then run following command
To regenerate the code, install **thrift-0.9.2** and change directory into Zeppelin source directory. and then run following command
```
thrift -out zeppelin-interpreter/src/main/java/ --gen java zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift
```
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
## Where to Start
You can find issues for [beginner](https://issues.apache.org/jira/browse/ZEPPELIN-924?jql=project%20%3D%20ZEPPELIN%20and%20status%20%3D%20Open%20and%20labels%20in%20\(beginner%2C%20newbie\)).
You can find issues for <a href="https://issues.apache.org/jira/browse/ZEPPELIN-981?jql=project%20%3D%20ZEPPELIN%20AND%20labels%20in%20(beginner%2C%20newbie)">beginner & newbie</a>
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/)
If you have any issues, create a ticket in [JIRA](https://issues.apache.org/jira/browse/ZEPPELIN).

41
docs/development/howtocontributewebsite.md
View file

@ -7,49 +7,52 @@ group: development
# Contributing to Apache Zeppelin ( Website )
## IMPORTANT
Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
<div id="toc"></div>
## Modifying the website
This page will give you an overview of how to build and contribute to the documentation of Apache Zeppelin.
The online documentation at [zeppelin.apache.org](https://zeppelin.apache.org/docs/latest/) is also generated from the files found here.
#### Getting the source code
Website is hosted in 'master' branch under `/docs/` dir.
> **NOTE :** Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
## Getting the source code
First of all, you need Zeppelin source code. The official location of Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git).
Documentation website is hosted in 'master' branch under `/docs/` dir.
### git access
First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git).
Get the source code on your development machine using git.
```
git clone git://git.apache.org/zeppelin.git
cd docs
```
Apache Zeppelin follows [Fork & Pull](https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) as a source control workflow.
If you want to not only build Zeppelin but also make any changes, then you need to fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request.
#### Build
### Build
To build, you'll need to install some prerequisites. Please check 'Build documentation' section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md#build-documentation).
You'll need to install some prerequisites to build the code. Please check [Build documentation](https://github.com/apache/zeppelin/blob/master/docs/README.md#build-documentation) section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md).
#### Run website in development mode
### Run website in development mode
While you're modifying website, you'll want to see preview of it. Please check 'Run website' section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md#run-website).
While you're modifying website, you might want to see preview of it. Please check [Run website](https://github.com/apache/zeppelin/blob/master/docs/README.md#run-website) section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md).
Then you'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser.
You'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser.
#### Making a Pull Request
### Making a Pull Request
When you are ready, just make a pull-request.
## Alternative way
You can directly edit .md files in `/docs/` dir at github's web interface and make pull-request immediatly.
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
You can directly edit `.md` files in `/docs/` directory at the web interface of github and make pull-request immediatly.
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/)
If you have any issues, create a ticket in [JIRA](https://issues.apache.org/jira/browse/ZEPPELIN).

66
docs/development/writingzeppelininterpreter.md
View file

@ -19,21 +19,25 @@ limitations under the License.
-->
{% include JB/setup %}
### What is Zeppelin Interpreter
# Writing a New Interpreter
Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need scala interpreter.
Every Interpreter belongs to an InterpreterGroup.
<div id="toc"></div>
## What is Apache Zeppelin Interpreter
Apache Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need a scala interpreter.
Every Interpreters belongs to an **InterpreterGroup**.
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/interpreter.png" />
InterpreterSetting is configuration of a given InterpreterGroup and a unit of start/stop interpreter.
All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via thrift.
[InterpreterSetting](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java) is configuration of a given [InterpreterGroup](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java) and a unit of start/stop interpreter.
All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via **[Thrift](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift)**.
In 'Separate Interpreter for each note' mode, new Interpreter instance will be created per notebook. But it still runs on the same JVM while they're in the same InterpreterSettings.
In 'Separate Interpreter(scoped / isolated) for each note' mode which you can see at the **Interpreter Setting** menu when you create a new interpreter, new interpreter instance will be created per notebook. But it still runs on the same JVM while they're in the same InterpreterSettings.
### Make your own Interpreter
## Make your own Interpreter
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should your jars under your interpreter directory with specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
@ -91,18 +95,18 @@ The name of the interpreter is what you later write to identify a paragraph whic
some interpreter specific code...
```
### Programming Languages for Interpreter
## Programming Languages for Interpreter
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.
1. Add the `mode-*.js` file to <code>[zeppelin-web/bower.json](https://github.com/apache/zeppelin/blob/master/zeppelin-web/bower.json)</code> ( when built, <code>[zeppelin-web/src/index.html](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/index.html)</code> will be changed automatically. ).
2. Add to the list of `editorMode` in <code>[zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js)</code> - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in <code>[zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js)</code>.
### Install your interpreter binary
## Install your interpreter binary
Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies.
@ -110,7 +114,7 @@ Once you have built your interpreter, you can place it under the interpreter dir
[ZEPPELIN_HOME]/interpreter/[INTERPRETER_NAME]/
```
### Configure your interpreter
## Configure your interpreter
To configure your interpreter you need to follow these steps:
@ -119,12 +123,12 @@ To configure your interpreter you need to follow these steps:
Property value is comma separated [INTERPRETER\_CLASS\_NAME].
For example,
```
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter</value>
</property>
```
```
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter</value>
</property>
```
2. Add your interpreter to the [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `zeppelin-site.xml`.
@ -133,11 +137,11 @@ To configure your interpreter you need to follow these steps:
4. In the interpreter page, click the `+Create` button and configure your interpreter properties.
Now you are done and ready to use your interpreter.
Note that the interpreters released with zeppelin have a [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
> **Note :** Interpreters released with zeppelin have a [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
### Use your interpreter
## Use your interpreter
#### 0.5.0
### 0.5.0
Inside of a notebook, `%[INTERPRETER_NAME]` directive will call your interpreter.
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
@ -150,8 +154,7 @@ val a = "My interpreter"
println(a)
```
<br />
#### 0.6.0 and later
### 0.6.0 and later
Inside of a notebook, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter.
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
@ -192,7 +195,7 @@ You can only omit your interpreter group when your interpreter group is selected
codes for myintp2
```
### Examples
## Examples
Checkout some interpreters released with Zeppelin by default.
@ -201,15 +204,16 @@ Checkout some interpreters released with Zeppelin by default.
- [shell](https://github.com/apache/zeppelin/tree/master/shell)
- [jdbc](https://github.com/apache/zeppelin/tree/master/jdbc)
### Contributing a new Interpreter to Zeppelin releases
## Contributing a new Interpreter to Zeppelin releases
We welcome contribution to a new interpreter. Please follow these few steps:
- First, check out the general contribution guide [here](./howtocontributewebsite.html).
- Follow the steps in "Make your own Interpreter" section above.
- Add your interpreter as in the "Configure your interpreter" section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template).
- Add tests! They are run by Travis for all changes and it is important that they are self-contained.
- First, check out the general contribution guide [here](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md).
- Follow the steps in [Make your own Interpreter](#make-your-own-interpreter) section above.
- Add your interpreter as in the [Configure your interpreter](#configure-your-interpreter) section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template).
- Add tests! They are run by [Travis](https://travis-ci.org/apache/zeppelin) for all changes and it is important that they are self-contained.
- Include your interpreter as a module in [`pom.xml`](https://github.com/apache/zeppelin/blob/master/pom.xml).
- Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`).
- Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE).
- Commit your changes and open a Pull Request on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing.
- Commit your changes and open a [Pull Request](https://github.com/apache/zeppelin/pulls) on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing.

29
docs/displaysystem/back-end-angular.md
View file

@ -19,15 +19,17 @@ limitations under the License.
-->
{% include JB/setup %}
# Back-end Angular API in Apache Zeppelin
## Back-end Angular API in Zeppelin
<div id="toc"></div>
## Overview
Angular display system treats output as a view template for [AngularJS](https://angularjs.org/).
It compiles templates and displays them inside of Zeppelin.
Zeppelin provides a gateway between your interpreter and your compiled **AngularJS view** templates.
It compiles templates and displays them inside of Apache Zeppelin. Zeppelin provides a gateway between your interpreter and your compiled **AngularJS view** templates.
Therefore, you can not only update scope variables from your interpreter but also watch them in the interpreter, which is JVM process.
## Basic Usage
### Print AngularJS view
To use angular display system, you should start with `%angular`.
@ -94,30 +96,25 @@ When the button is clicked, you'll see both `run` and `numWatched` are increment
## Let's make it Simpler and more Intuitive
In this section, we will introduce a simpler and more intuitive way of using **Angular Display System** in Zeppelin.
### How can we use it?
Here are some usages.
#### Import
##### - In notebook scope
### Import
```scala
// In notebook scope
import org.apache.zeppelin.display.angular.notebookscope._
import AngularElem._
```
##### - In paragraph scope
```scala
// In paragraph scope
import org.apache.zeppelin.display.angular.paragraphscope._
import AngularElem._
```
#### Display Element
### Display Element
```scala
// automatically convert to string and print with %angular display system directive in front.
<div><div>.display
```
#### Event Handler
### Event Handler
```scala
// on click
<div></div>.onClick(() => {
@ -135,7 +132,7 @@ import AngularElem._
}).display
```
#### Bind Model
### Bind Model
```scala
// bind model
<div></div>.model("myModel").display
@ -144,7 +141,7 @@ import AngularElem._
<div></div>.model("myModel", initialValue).display
```
#### Interact with Model
### Interact with Model
```scala
// read model
AngularModel("myModel")()

4
docs/displaysystem/basicdisplaysystem.md
View file

@ -19,6 +19,10 @@ limitations under the License.
-->
{% include JB/setup %}
# Basic Display System in Apache Zeppelin
<div id="toc"></div>
## Text
By default, Apache Zeppelin prints interpreter responce as a plain text using `text` display system.

25
docs/displaysystem/front-end-angular.md
View file

@ -19,18 +19,18 @@ limitations under the License.
-->
{% include JB/setup %}
# Front-end Angular API in Apache Zeppelin
## Front-end Angular API in Zeppelin
<div id="toc"></div>
In addition to the back-end API to handle Angular objects binding, Zeppelin also exposes a simple AngularJS **`z`** object on the front-end side to expose the same capabilities.
## Basic Usage
In addition to the [back-end API](./back-end-angular.html) to handle Angular objects binding, Apache Zeppelin also exposes a simple AngularJS <code>**z**</code> object on the front-end side to expose the same capabilities.
This <code>**z**</code> object is accessible in the Angular isolated scope for each paragraph.
This **`z`** object is accessible in the Angular isolated scope for each paragraph.
<br />
### Bind / Unbind Variables
Through the **`z`**, you can bind / unbind variables to **AngularJS view**
Through the **`z`**, you can bind / unbind variables to **AngularJS view**.
Bind a value to an angular object and a **mandatory** target paragraph:
```html
@ -68,9 +68,10 @@ Unbind/remove a value from angular object and a **mandatory** target paragraph:
The signature for the **`z.angularBind() / z.angularUnbind()`** functions are:
```javascript
// Bind
z.angularBind(angularObjectName, angularObjectValue, paragraphId);
// Unbind
z.angularUnbind(angularObjectName, angularObjectValue, paragraphId);
```
@ -100,24 +101,24 @@ You can also trigger paragraph execution by calling **`z.runParagraph()`** funct
<img src="../assets/themes/zeppelin/img/screenshots/z_runParagraph.gif" />
<br />
### Overriding dynamic form with Angular Object
## Overriding dynamic form with Angular Object
The front-end Angular Interaction API has been designed to offer richer form capabilities and variable binding. With the existing **Dynamic Form** system you can already create input text, select and checkbox forms but the choice is rather limited and the look & feel cannot be changed.
The idea is to create a custom form using plain HTML/AngularJS code and bind actions on this form to push/remove Angular variables to targeted paragraphs using this new API.
Consequently if you use the **Dynamic Form** syntax in a paragraph and there is a bound Angular object having the same name as the _${formName}_, the Angular object will have higher priority and the **Dynamic Form** will not be displayed. Example:
Consequently if you use the **Dynamic Form** syntax in a paragraph and there is a bound Angular object having the same name as the `${formName}`, the Angular object will have higher priority and the **Dynamic Form** will not be displayed. Example:
<img src="../assets/themes/zeppelin/img/screenshots/z_angularJs_overriding_dynamic_form.gif" />
<br />
### Feature matrix comparison
## Feature matrix comparison
How does the front-end AngularJS API compares to the back-end API ? Below is a comparison matrix for both APIs:
How does the front-end AngularJS API compares to the [back-end API](./back-end-angular.html) ? Below is a comparison matrix for both APIs:
<table>
<table class="table-configuration">
<thead>
<tr>
<th>Actions</th>

24
docs/index.md
View file

@ -33,7 +33,7 @@ limitations under the License.
</ul>
</div>
<div class="col-md-6" style="padding:0">
<img class="img-responsive" style="border: 1px solid #ecf0f1;" src="/assets/themes/zeppelin/img/notebook.png" />
<img class="img-responsive" style="border: 1px solid #ecf0f1;" src="./assets/themes/zeppelin/img/notebook.png" />
</div>
</div>
@ -42,14 +42,14 @@ limitations under the License.
[Apache Zeppelin interpreter](./manual/interpreters.html) concept allows any language/data-processing-backend to be plugged into Zeppelin.
Currently Apache Zeppelin supports many interpreters such as Apache Spark, Python, JDBC, Markdown and Shell.
<img class="img-responsive" width="500px" style="margin:0 auto; padding: 26px;" src="/assets/themes/zeppelin/img/available_interpreters.png" />
<img class="img-responsive" width="500px" style="margin:0 auto; padding: 26px;" src="./assets/themes/zeppelin/img/available_interpreters.png" />
Adding new language-backend is really simple. Learn [how to create your own interpreter](./development/writingzeppelininterpreter.html#make-your-own-interpreter).
#### Apache Spark integration
Especially, Apache Zeppelin provides built-in [Apache Spark](http://spark.apache.org/) integration. You don't need to build a separate module, plugin or library for it.
<img class="img-responsive" src="/assets/themes/zeppelin/img/spark_logo.jpg" width="140px" />
<img class="img-responsive" src="./assets/themes/zeppelin/img/spark_logo.jpg" width="140px" />
Apache Zeppelin with Spark integration provides
@ -66,10 +66,10 @@ Some basic charts are already included in Apache Zeppelin. Visualizations are no
<div class="row">
<div class="col-md-6">
<img class="img-responsive" src="/assets/themes/zeppelin/img/graph1.png" />
<img class="img-responsive" src="./assets/themes/zeppelin/img/graph1.png" />
</div>
<div class="col-md-6">
<img class="img-responsive" src="/assets/themes/zeppelin/img/graph2.png" />
<img class="img-responsive" src="./assets/themes/zeppelin/img/graph2.png" />
</div>
</div>
@ -79,7 +79,7 @@ Apache Zeppelin aggregates values and displays them in pivot chart with simple d
<div class="row">
<div class="col-md-12">
<img class="img-responsive" style="margin: 16px auto;" src="/assets/themes/zeppelin/img/screenshots/pivot.png" width="480px" />
<img class="img-responsive" style="margin: 16px auto;" src="./assets/themes/zeppelin/img/screenshots/pivot.png" width="480px" />
</div>
</div>
@ -91,7 +91,7 @@ Learn more about [display systems](#display-system) in Apache Zeppelin.
Apache Zeppelin can dynamically create some input forms in your notebook.
<div class="row">
<div class="col-md-12">
<img class="img-responsive" style="margin: 16px auto;" src="/assets/themes/zeppelin/img/screenshots/dynamicform.png" />
<img class="img-responsive" style="margin: 16px auto;" src="./assets/themes/zeppelin/img/screenshots/dynamicform.png" />
</div>
</div>
Learn more about [Dynamic Forms](./manual/dynamicform.html).
@ -102,7 +102,7 @@ Your notebook URL can be shared among collaborators. Then Apache Zeppelin will b
<div class="row">
<div class="col-md-12">
<img class="img-responsive" style="margin: 20px auto" src="/assets/themes/zeppelin/img/screenshots/publish.png" width="650px"/>
<img class="img-responsive" style="margin: 20px auto" src="./assets/themes/zeppelin/img/screenshots/publish.png" width="650px"/>
</div>
</div>
@ -113,7 +113,7 @@ If you want to learn more about this feature, please visit [this page](./manual/
<br />
## 100% Opensource
<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="/assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="./assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](./development/howtocontribute.html).
Apache Zeppelin has a very active development community.
@ -124,9 +124,9 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
####Quick Start
* Getting Started
* [Quick Start](./install/install.html) for basic instructions on installing Zeppelin
* [Configuration](./install/install.html#zeppelin-configuration) lists for Zeppelin
* [Explore Apache Zeppelin UI](./quickstart/explorezeppelinui.html): basic components of Zeppelin home
* [Quick Start](./install/install.html) for basic instructions on installing Apache Zeppelin
* [Configuration](./install/install.html#apache-zeppelin-configuration) lists for Apache Zeppelin
* [Explore Apache Zeppelin UI](./quickstart/explorezeppelinui.html): basic components of Apache Zeppelin home
* [Tutorial](./quickstart/tutorial.html): a short walk-through tutorial that uses Apache Spark backend
* Basic Feature Guide
* [Dynamic Form](./manual/dynamicform.html): a step by step guide for creating dynamic forms

166
docs/install/install.md
View file

@ -19,38 +19,108 @@ limitations under the License.
-->
{% include JB/setup %}
## Zeppelin Installation
Welcome to your first trial to explore Zeppelin!
# Quick Start
Welcome to your first trial to explore Apache Zeppelin!
This page will help you to get started and here is the list of topics covered.
In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below.
<div id="toc"></div>
### Install with Binary Package
## Installation
If you want to install Zeppelin with latest binary package, please visit [this page](http://zeppelin.apache.org/download.html).
Apache Zeppelin officially supports and is tested on next environments.
### Build from Zeppelin Source
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>Oracle JDK</td>
<td>1.7 <br /> (set <code>JAVA_HOME</code>)</td>
</tr>
<tr>
<td>OS</td>
<td>Mac OSX <br /> Ubuntu 14.X <br /> CentOS 6.X <br /> Windows 7 Pro SP1</td>
</tr>
</table>
You can also build Zeppelin from the source.
There are two options to install Apache Zeppelin on your machine. One is [downloading pre-built binary package](#downloading-binary-package) from the archive.
You can download not only the latest stable version but also the older one if you need.
The other option is [building from the source](#building-from-source).
Although it can be unstable somehow since it is on development status, you can explore newly added feature and change it as you want.
#### Prerequisites for build
* Java 1.7
* Git
* Maven(3.1.x or higher)
* Node.js Package Manager
### Downloading Binary Package
If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/zeppelin/blob/master/README.md) for the details.
If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html).
If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`.
After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section.
Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation.
[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example.
### Building from Source
If you want to build from the source, the software below needs to be installed on your system.
## Zeppelin Start / Stop
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>Git</td>
<td></td>
</tr>
<tr>
<td>Maven</td>
<td>3.1.x or higher</td>
</tr>
</table>
If you don't have it installed yet, please check [Before Build](https://github.com/apache/zeppelin/blob/master/README.md#before-build) section and follow step by step instructions from there.
####1. Clone Apache Zeppelin repository
```
git clone https://github.com/apache/zeppelin.git
```
####2. Build source with options
Each interpreters requires different build options. For the further information about options, please see [Build](https://github.com/apache/zeppelin#build) section.
```
mvn clean package -DskipTests [Options]
```
Here are some examples with several options
```
# basic build
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests
# with CDH
mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests
# with MapR
mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
```
For the further information about building with source, please see [README.md](https://github.com/apache/zeppelin/blob/master/README.md) in Zeppelin repository.
## Starting Apache Zeppelin with Command Line
#### Start Zeppelin
```
bin/zeppelin-daemon.sh start
```
If you are using Windows
```
bin\zeppelin.cmd
```
After successful start, visit [http://localhost:8080](http://localhost:8080) with your web browser.
#### Stop Zeppelin
@ -59,21 +129,28 @@ After successful start, visit [http://localhost:8080](http://localhost:8080) wit
bin/zeppelin-daemon.sh stop
```
#### Start Zeppelin with a service manager such as upstart
#### (Optional) Start Apache Zeppelin with a service manager
Zeppelin can auto start as a service with an init script, such as services managed by upstart.
> **Note :** The below description was written based on Ubuntu Linux.
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
This example has been tested with Ubuntu Linux.
Apache Zeppelin can be auto started as a service with an init script, such as services managed by **upstart**.
The following is an example of upstart script to be saved as `/etc/init/zeppelin.conf`
This also allows the service to be managed with commands such as
`sudo service zeppelin start`
`sudo service zeppelin stop`
`sudo service zeppelin restart`
```
sudo service zeppelin start
sudo service zeppelin stop
sudo service zeppelin restart
```
Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart`
Other service managers could use a similar approach with the `upstart` argument passed to the `zeppelin-daemon.sh` script.
##### zeppelin.conf
```
bin/zeppelin-daemon.sh upstart
```
**zeppelin.conf**
```
description "zeppelin"
@ -93,15 +170,24 @@ chdir /usr/share/zeppelin
exec bin/zeppelin-daemon.sh upstart
```
#### Running on Windows
## What is the next?
Congratulation on your successful Apache Zeppelin installation! Here are two next steps you might need.
```
bin\zeppelin.cmd
```
#### If you are new to Apache Zeppelin
* For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html).
* After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend.
* If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration).
#### If you need more information about Spark or JDBC interpreter setting
* Apache Zeppelin provides deep integration with [Apache Spark](http://spark.apache.org/). For the further informtation, see [Spark Interpreter for Apache Zeppelin](../interpreter/spark.html).
* Also, you can use generic JDBC connections in Apache Zeppelin. Go to [Generic JDBC Interpreter for Apache Zeppelin](../interpreter/jdbc.html).
#### If you are in multi-user environment
* You can set permissions for your notebooks and secure data resource in multi-user environment. Go to **More** -> **Security** section.
## Apache Zeppelin Configuration
## Zeppelin Configuration
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
You can configure Apache Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
<table class="table-configuration">
<tr>
@ -210,13 +296,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN</td>
<td>zeppelin.notebook.homescreen</td>
<td></td>
<td>A notebook id displayed in Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
<td>A notebook id displayed in Apache Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE</td>
<td>zeppelin.notebook.homescreen.hide</td>
<td>false</td>
<td>This value can be "true" when to hide the notebook id set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
<td>This value can be "true" when to hide the notebook id set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Apache Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
</tr>
<tr>
<td>ZEPPELIN_WAR_TEMPDIR</td>
@ -228,13 +314,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_DIR</td>
<td>zeppelin.notebook.dir</td>
<td>notebook</td>
<td>The root directory where Zeppelin notebook directories are saved</td>
<td>The root directory where notebook directories are saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_BUCKET</td>
<td>zeppelin.notebook.s3.bucket</td>
<td>zeppelin</td>
<td>S3 Bucket where Zeppelin notebook files will be saved</td>
<td>S3 Bucket where notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_USER</td>
@ -270,7 +356,7 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Share where the Zeppelin notebook files will be saved</td>
<td>Share where the notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
@ -291,13 +377,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />
...
</td>
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in Zeppelin notebook paragraph. </td>
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in notebook paragraph. </td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETER_DIR</td>
<td>zeppelin.interpreter.dir</td>
<td>interpreter</td>
<td>Zeppelin interpreter directory</td>
<td>Interpreter directory</td>
</tr>
<tr>
<td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td>
@ -305,4 +391,4 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>1024000</td>
<td>Size in characters of the maximum text message to be received by websocket.</td>
</tr>
</table>
</table>

10
docs/install/upgrade.md
View file

@ -19,12 +19,14 @@ limitations under the License.
-->
{% include JB/setup %}
## Manual upgrade procedure for Zeppelin
# Manual upgrade procedure for Zeppelin
<div id="toc"></div>
Basically, newer version of Zeppelin works with previous version notebook directory and configurations.
So, copying `notebook` and `conf` directory should be enough.
### Instructions
## Instructions
1. Stop Zeppelin
```
@ -33,7 +35,7 @@ So, copying `notebook` and `conf` directory should be enough.
1. Copy your `notebook` and `conf` directory into a backup directory
1. Download newer version of Zeppelin and Install. See [Install page](./install.html)
1. Download newer version of Zeppelin and Install. See [Install page](./install.html#installation).
1. Copy backup `notebook` and `conf` directory into newer version of Zeppelin `notebook` and `conf` directory
@ -41,4 +43,4 @@ So, copying `notebook` and `conf` directory should be enough.
```
bin/zeppelin-daemon.sh start
```
```

28
docs/install/virtual_machine.md
View file

@ -19,8 +19,11 @@ limitations under the License.
-->
{% include JB/setup %}
# Vagrant Virtual Machine for Apache Zeppelin
## Vagrant Virtual Machine for Apache Zeppelin
<div id="toc"></div>
## Overview
Apache Zeppelin distribution includes a scripts directory
@ -30,11 +33,11 @@ This script creates a virtual machine that launches a repeatable, known set of c
For PySpark users, this script includes several helpful [Python Libraries](#python-extras).
For SparkR users, this script includes several helpful [R Libraries](#r-extras).
####Installing the required components to launch a virtual machine.
### Prerequisites
This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box"). All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
## Create a Zeppelin Ready VM
If you are running Windows and don't yet have python installed, [install Python 2.7.x](https://www.python.org/downloads/release/python-2710/) first.
@ -60,9 +63,15 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
```
### Building Zeppelin
## Building Zeppelin
You can now `git clone git://git.apache.org/zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now
```
git clone git://git.apache.org/zeppelin.git
```
into a directory on your host machine, or directly in your virtual machine.
Cloning Zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
@ -74,14 +83,13 @@ By default, Vagrant will share your project directory (the directory with the Va
`cd /vagrant/zeppelin`
### What's in this VM?
## What's in this VM?
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
The virtual machine consists of:
- Ubuntu Server 14.04 LTS
@ -96,7 +104,7 @@ The virtual machine consists of:
- Python addons: pip, matplotlib, scipy, numpy, pandas
- [R](https://www.r-project.org/) and R Packages required to run the R Interpreter and the related R tutorial notebook, including: Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table
### How to build & run Zeppelin
## How to build & run Zeppelin
This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine. The following build steps will also include Python and R support via PySpark and SparkR:
@ -111,7 +119,7 @@ On your host machine browse to `http://localhost:8080/`
If you [turned off port forwarding](#tweaking-the-virtual-machine) in the `Vagrantfile` browse to `http://192.168.51.52:8080`
### Tweaking the Virtual Machine
## Tweaking the Virtual Machine
If you plan to run this virtual machine along side other Vagrant images, you may wish to bind the virtual machine to a specific IP address, and not use port fowarding from your local host.
@ -125,7 +133,7 @@ config.vm.network "private_network", ip: "192.168.51.52"
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
## Extras
### Python Extras
With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available. Create a pyspark notebook, and try the below code.

6
docs/interpreter/alluxio.md
View file

@ -6,7 +6,11 @@ group: manual
---
{% include JB/setup %}
## Alluxio Interpreter for Apache Zeppelin
# Alluxio Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Alluxio](http://alluxio.org/) is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks.
## Configuration

4
docs/interpreter/cassandra.md
View file

@ -6,7 +6,9 @@ group: manual
---
{% include JB/setup %}
## Cassandra CQL Interpreter for Apache Zeppelin
# Cassandra CQL Interpreter for Apache Zeppelin
<div id="toc"></div>
<table class="table-configuration">
<tr>

6
docs/interpreter/elasticsearch.md
View file

@ -6,7 +6,11 @@ group: manual
---
{% include JB/setup %}
## Elasticsearch Interpreter for Apache Zeppelin
# Elasticsearch Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Elasticsearch](https://www.elastic.co/products/elasticsearch) is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements.
## Configuration

6
docs/interpreter/flink.md
View file

@ -6,7 +6,11 @@ group: manual
---
{% include JB/setup %}
## Flink interpreter for Apache Zeppelin
# Flink interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing. Flink’s core is a streaming dataflow engine that provides data distribution, communication, and fault tolerance for distributed computations over data streams. Flink also builds batch processing on top of the streaming engine, overlaying native iteration support, managed memory, and program optimization.
## How to start local Flink cluster, to test the interpreter

30
docs/interpreter/geode.md
View file

@ -6,7 +6,11 @@ group: manual
---
{% include JB/setup %}
## Geode/Gemfire OQL Interpreter for Apache Zeppelin
# Geode/Gemfire OQL Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
<table class="table-configuration">
<tr>
<th>Name</th>
@ -33,7 +37,7 @@ This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Objec
This [Video Tutorial](https://www.youtube.com/watch?v=zvzzA9GXu3Q) illustrates some of the features provided by the `Geode Interpreter`.
### Create Interpreter
## Create Interpreter
By default Zeppelin creates one `Geode/OQL` instance. You can remove it or create more instances.
Multiple Geode instances can be created, each configured to the same or different backend Geode cluster. But over time a `Notebook` can have only one Geode interpreter instance `bound`. That means you _cannot_ connect to different Geode clusters in the same `Notebook`. This is a known Zeppelin limitation.
@ -42,10 +46,10 @@ To create new Geode instance open the `Interpreter` section and click the `+Crea
> Note: The `Name` of the instance is used only to distinguish the instances while binding them to the `Notebook`. The `Name` is irrelevant inside the `Notebook`. In the `Notebook` you must use `%geode.oql` tag.
### Bind to Notebook
## Bind to Notebook
In the `Notebook` click on the `settings` icon in the top right corner. The select/deselect the interpreters to be bound with the `Notebook`.
### Configuration
## Configuration
You can modify the configuration of the Geode from the `Interpreter` section. The Geode interpreter expresses the following properties:
<table class="table-configuration">
@ -71,12 +75,12 @@ You can modify the configuration of the Geode from the `Interpreter` section. T
</tr>
</table>
### How to use
## How to use
> *Tip 1: Use (CTRL + .) for OQL auto-completion.*
> *Tip 2: Always start the paragraphs with the full `%geode.oql` prefix tag! The short notation: `%geode` would still be able run the OQL queries but the syntax highlighting and the auto-completions will be disabled.*
#### Create / Destroy Regions
### Create / Destroy Regions
The OQL specification does not support [Geode Regions](https://cwiki.apache.org/confluence/display/GEODE/Index#Index-MainConceptsandComponents) mutation operations. To `create`/`destroy` regions one should use the [GFSH](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/chapter_overview.html) shell tool instead. In the following it is assumed that the GFSH is colocated with Zeppelin server.
```bash
@ -97,7 +101,7 @@ EOF
Above snippet re-creates two regions: `regionEmployee` and `regionCompany`. Note that you have to explicitly specify the locator host and port. The values should match those you have used in the Geode Interpreter configuration. Comprehensive list of [GFSH Commands by Functional Area](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/gfsh_quick_reference.html).
#### Basic OQL
### Basic OQL
```sql
%geode.oql
SELECT count(*) FROM /regionEmployee
@ -136,7 +140,7 @@ SELECT e.key, e.value FROM /regionEmployee.entrySet e
> Note: You can have multiple queries in the same paragraph but only the result from the first is displayed. [[1](https://issues.apache.org/jira/browse/ZEPPELIN-178)], [[2](https://issues.apache.org/jira/browse/ZEPPELIN-212)].
#### GFSH Commands From The Shell
### GFSH Commands From The Shell
Use the Shell Interpreter (`%sh`) to run OQL commands form the command line:
```bash
@ -145,7 +149,7 @@ source /etc/geode/conf/geode-env.sh
gfsh -e "connect" -e "list members"
```
#### Apply Zeppelin Dynamic Forms
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your OQL queries. You can use both the `text input` and `select form` parameterization features
```sql
@ -153,7 +157,10 @@ You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your
SELECT * FROM /regionEmployee e WHERE e.employeeId > ${Id}
```
#### Geode REST API
### Auto-completion
The Geode Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window.
## Geode REST API
To list the defined regions you can use the [Geode REST API](http://geode-docs.cfapps.io/docs/geode_rest/chapter_overview.html):
```
@ -182,6 +189,3 @@ http://<geode server hostname>phd1.localdomain:8484/gemfire-api/v1/
http-service-port=8484
start-dev-rest-api=true
```
### Auto-completion
The Geode Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window.

12
docs/interpreter/hbase.md
View file

@ -6,16 +6,22 @@ group: manual
---
{% include JB/setup %}
## HBase Shell Interpreter for Apache Zeppelin
# HBase Shell Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed.
To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart)
To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart).
## HBase release supported
By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1.1.x releases, use the following build command:
```bash
# HBase 1.1.4
mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0
```
To work with HBase 1.2.0+, use the following build command:
```bash
@ -94,4 +100,4 @@ And then to put data into that table
put 'test', 'row1', 'cf:a', 'value1'
```
For more information on all commands available, refer to [HBase shell commands](https://learnhbase.wordpress.com/2013/03/02/hbase-shell-commands/)
For more information on all commands available, refer to [HBase shell commands](https://learnhbase.wordpress.com/2013/03/02/hbase-shell-commands/).

13
docs/interpreter/hdfs.md
View file

@ -6,8 +6,11 @@ group: manual
---
{% include JB/setup %}
## HDFS File System Interpreter for Apache Zeppelin
# HDFS File System Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Hadoop File System](http://hadoop.apache.org/) is a distributed, fault tolerant file system part of the hadoop project and is often used as storage for distributed processing engines like [Hadoop MapReduce](http://hadoop.apache.org/) and [Apache Spark](http://spark.apache.org/) or underlying file systems like [Alluxio](http://www.alluxio.org/).
## Configuration
@ -44,13 +47,17 @@ It supports the basic shell file commands applied to HDFS, it currently only sup
> **Tip :** Use ( Ctrl + . ) for autocompletion.
### Create Interpreter
## Create Interpreter
In a notebook, to enable the **HDFS** interpreter, click the **Gear** icon and select **HDFS**.
#### WebHDFS REST API
## WebHDFS REST API
You can confirm that you're able to access the WebHDFS API by running a curl command against the WebHDFS end point provided to the interpreter.
Here is an example:
```bash
$> curl "http://localhost:50070/webhdfs/v1/?op=LISTSTATUS"
```

10
docs/interpreter/hive.md
View file

@ -6,8 +6,9 @@ group: manual
---
{% include JB/setup %}
## Hive Interpreter for Apache Zeppelin
The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL.
# Hive Interpreter for Apache Zeppelin
<div id="toc"></div>
## Important Notice
Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can use Hive Interpreter by using JDBC Interpreter with same functionality. See the example below of settings and dependencies.
@ -52,7 +53,6 @@ Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can us
</tr>
</table>
----
### Configuration
<table class="table-configuration">
@ -115,6 +115,10 @@ Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can us
This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
## Overview
The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL.
## How to use
Basically, you can use

14
docs/interpreter/ignite.md
View file

@ -6,16 +6,18 @@ group: manual
---
{% include JB/setup %}
## Ignite Interpreter for Apache Zeppelin
# Ignite Interpreter for Apache Zeppelin
### Overview
<div id="toc"></div>
## Overview
[Apache Ignite](https://ignite.apache.org/) In-Memory Data Fabric is a high-performance, integrated and distributed in-memory platform for computing and transacting on large-scale data sets in real-time, orders of magnitude faster than possible with traditional disk-based or flash technologies.
![Apache Ignite](../assets/themes/zeppelin/img/docs-img/ignite-logo.png)
You can use Zeppelin to retrieve distributed data from cache using Ignite SQL interpreter. Moreover, Ignite interpreter allows you to execute any Scala code in cases when SQL doesn't fit to your requirements. For example, you can populate data into your caches or execute distributed computations.
### Installing and Running Ignite example
## Installing and Running Ignite example
In order to use Ignite interpreters, you may install Apache Ignite in some simple steps:
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zeppelin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
@ -31,7 +33,7 @@ In order to use Ignite interpreters, you may install Apache Ignite in some simpl
$ nohup java -jar </path/to/your Jar file name>
```
### Configuring Ignite Interpreter
## Configuring Ignite Interpreter
At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Zeppelin provides these properties for Ignite.
<table class="table-configuration">
@ -69,14 +71,14 @@ At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Z
![Configuration of Ignite Interpreter](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-setting.png)
### Interpreter Binding for Zeppelin Notebook
## How to use
After configuring Ignite interpreter, create your own notebook. Then you can bind interpreters like below image.
![Binding Interpreters](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-binding.png)
For more interpreter binding information see [here](http://zeppelin.apache.org/docs/manual/interpreters.html).
### How to use Ignite SQL interpreter
### Ignite SQL interpreter
In order to execute SQL query, use ` %ignite.ignitesql ` prefix. <br>
Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ).
For example, you can select top 10 words in the words cache using the following query

88
docs/interpreter/jdbc.md
View file

@ -7,7 +7,11 @@ group: manual
{% include JB/setup %}
## Generic JDBC Interpreter for Apache Zeppelin
# Generic JDBC Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
This interpreter lets you create a JDBC connection to any data source, by now it has been tested with:
@ -16,15 +20,14 @@ This interpreter lets you create a JDBC connection to any data source, by now it
* MariaDB
* Redshift
* Apache Hive
* Apache Drill
* Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver)
* Apache Tajo
* Apache Phoenix
* Apache Drill (Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driverde* Apache Tajo
If someone else used another database please report how it works to improve functionality.
### Create Interpreter
## Create Interpreter
When create a interpreter by default use PostgreSQL with the next properties:
When you create a interpreter by default use PostgreSQL with the next properties:
<table class="table-configuration">
<tr>
@ -55,7 +58,7 @@ When create a interpreter by default use PostgreSQL with the next properties:
It is not necessary to add driver jar to the classpath for PostgreSQL as it is included in Zeppelin.
#### Simple connection
### Simple connection
Prior to creating the interpreter it is necessary to add maven coordinate or path of the JDBC driver to the Zeppelin classpath. To do this you must edit dependencies artifact(ex. `mysql:mysql-connector-java:5.1.38`) in interpreter menu as shown:
@ -94,7 +97,7 @@ To create the interpreter you need to specify connection parameters as shown in
</tr>
</table>
#### Multiple connections
### Multiple connections
JDBC interpreter also allows connections to multiple data sources. It is necessary to set a prefix for each connection to reference it in the paragraph in the form of `%jdbc(prefix)`. Before you create the interpreter it is necessary to add each driver's maven coordinates or JDBC driver's jar file path to the Zeppelin classpath. To do this you must edit the dependencies of JDBC interpreter in interpreter menu as following:
@ -150,10 +153,10 @@ You can add all the jars you need to make multiple connections into the same JDB
</table>
### Bind to Notebook
## Bind to Notebook
In the `Notebook` click on the `settings` icon at the top-right corner. Use select/deselect to specify the interpreters to be used in the `Notebook`.
### More Properties
## More Properties
You can modify the interpreter configuration in the `Interpreter` section. The most common properties are as follows, but you can specify other properties that need to be connected.
<table class="table-configuration">
@ -196,9 +199,11 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
</tr>
</table>
### Examples
#### Hive
##### Properties
## Examples
### Hive
#### Properties
<table class="table-configuration">
<tr>
<th>Name</th>
@ -221,7 +226,8 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
<td>hive_password</td>
</tr>
</table>
##### Dependencies
#### Dependencies
<table class="table-configuration">
<tr>
<th>Artifact</th>
@ -236,8 +242,45 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
<td></td>
</tr>
</table>
#### Tajo
##### Properties
### Phoenix
#### Properties
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>phoenix.driver</td>
<td>org.apache.phoenix.jdbc.PhoenixDriver</td>
</tr>
<tr>
<td>phoenix.url</td>
<td>jdbc:phoenix:localhost:2181:/hbase-unsecure</td>
</tr>
<tr>
<td>phoenix.user</td>
<td>phoenix_user</td>
</tr>
<tr>
<td>phoenix.password</td>
<td>phoenix_password</td>
</tr>
</table>
#### Dependencies
<table class="table-configuration">
<tr>
<th>Artifact</th>
<th>Excludes</th>
</tr>
<tr>
<td>org.apache.phoenix:phoenix-core:4.4.0-HBase-1.0</td>
<td></td>
</tr>
</table>
### Tajo
#### Properties
<table class="table-configuration">
<tr>
<th>Name</th>
@ -252,7 +295,8 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
<td>jdbc:tajo://localhost:26002/default</td>
</tr>
</table>
##### Dependencies
#### Dependencies
<table class="table-configuration">
<tr>
<th>Artifact</th>
@ -264,9 +308,9 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
</tr>
</table>
### How to use
## How to use
#### Reference in paragraph
### Reference in paragraph
Start the paragraphs with the `%jdbc`, this will use the `default` prefix for connection. If you want to use other connection you should specify the prefix of it as follows `%jdbc(prefix)`:
@ -275,14 +319,16 @@ Start the paragraphs with the `%jdbc`, this will use the `default` prefix for co
SELECT * FROM db_name;
```
or
```sql
%jdbc(prefix)
SELECT * FROM db_name;
```
#### Apply Zeppelin Dynamic Forms
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
@ -293,5 +339,5 @@ FROM demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
```
### Bugs & Contacts
## Bugs & Reporting
If you find a bug for this interpreter, please create a [JIRA]( https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN) ticket.

12
docs/interpreter/lens.md
View file

@ -6,14 +6,16 @@ group: manual
---
{% include JB/setup %}
## Lens Interpreter for Apache Zeppelin
# Lens Interpreter for Apache Zeppelin
### Overview
<div id="toc"></div>
## Overview
[Apache Lens](https://lens.apache.org/) provides an Unified Analytics interface. Lens aims to cut the Data Analytics silos by providing a single view of data across multiple tiered data stores and optimal execution environment for the analytical query. It seamlessly integrates Hadoop with traditional data warehouses to appear like one.
![Apache Lens](../assets/themes/zeppelin/img/docs-img/lens-logo.png)
### Installing and Running Lens
## Installing and Running Lens
In order to use Lens interpreters, you may install Apache Lens in some simple steps:
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
@ -24,7 +26,7 @@ In order to use Lens interpreters, you may install Apache Lens in some simple st
./bin/lens-ctl start (or stop)
```
### Configuring Lens Interpreter
## Configuring Lens Interpreter
At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
<table class="table-configuration">
@ -163,7 +165,7 @@ query execute cube select customer_city_name, product_details.description, produ
These are just examples that provided in advance by Lens. If you want to explore whole tutorials of Lens, see the [tutorial video](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
### Lens UI Service
## Lens UI Service
Lens also provides web UI service. Once the server starts up, you can open the service on http://serverhost:19999/index.html and browse. You may also check the structure that you made and use query easily here.
![Lens UI Service](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png)

21
docs/interpreter/livy.md
View file

@ -6,8 +6,12 @@ group: manual
---
{% include JB/setup %}
## Livy Interpreter for Apache Zeppelin
Livy is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN.
# Livy Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Livy](http://livy.io/) is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN.
* Interactive Scala, Python and R shells
* Batch submissions in Scala, Java, Python
@ -16,13 +20,12 @@ Livy is an open source REST interface for interacting with Spark from anywhere.
* Does not require any code change to your programs
### Requirements
Additional requirements for the Livy interpreter are:
* Spark 1.3 or above.
* Livy server.
### Configuration
## Configuration
We added some common configurations for spark, and you can set any configuration you want.
This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties.
And instead of starting property with `spark.` it should be replaced with `livy.spark.`.
@ -101,8 +104,6 @@ Example: `spark.master` to `livy.spark.master`
</tr>
</table>
## How to use
Basically, you can use
@ -136,7 +137,7 @@ hello("livy")
When Zeppelin server is running with authentication enabled, then this interpreter utilizes Livy’s user impersonation feature i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server.
### Apply Zeppelin Dynamic Forms
## Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features.
```
@ -159,7 +160,7 @@ The session would have timed out, you may need to restart the interpreter.
> Blacklisted configuration values in session config: spark.master
edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line.
Edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line.
if you choose to work on livy in `apps/spark/java` directory in https://github.com/cloudera/hue ,
copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`
If you choose to work on livy in `apps/spark/java` directory in [https://github.com/cloudera/hue](https://github.com/cloudera/hue),
copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`.

8
docs/interpreter/markdown.md
View file

@ -6,9 +6,11 @@ group: manual
---
{% include JB/setup %}
## Markdown Interpreter for Apache Zeppelin
# Markdown Interpreter for Apache Zeppelin
### Overview
<div id="toc"></div>
## Overview
[Markdown](http://daringfireball.net/projects/markdown/) is a plain text formatting syntax designed so that it can be converted to HTML.
Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/).
In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to invoke the Markdown interpreter and generate static html from Markdown plain text.
@ -17,7 +19,7 @@ In Zeppelin, Markdown interpreter is enabled by default.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="60%" />
### Example
## Example
The following example demonstrates the basic usage of Markdown in a Zeppelin notebook.
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />

46
docs/interpreter/postgresql.md
View file

@ -6,7 +6,12 @@ group: manual
---
{% include JB/setup %}
# PostgreSQL, Apache HAWQ (incubating) Interpreter for Apache Zeppelin
<div id="toc"></div>
## Important Notice
Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You can use Postgresql by using JDBC Interpreter with same functionality. See the example below of settings and dependencies.
### Properties
@ -44,10 +49,19 @@ Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You
<td></td>
</tr>
</table>
---
----
## Overview
## PostgreSQL, HAWQ Interpreter for Apache Zeppelin
[<img align="right" src="http://img.youtube.com/vi/wqXXQhJ5Uk8/0.jpg" alt="zeppelin-view" hspace="10" width="250"></img>](https://www.youtube.com/watch?v=wqXXQhJ5Uk8)
This interpreter seamlessly supports the following SQL data processing engines:
* [PostgreSQL](http://www.postgresql.org/) - OSS, Object-relational database management system (ORDBMS)
* [pache HAWQ (incubating)](http://hawq.incubator.apache.org/) - Powerful open source SQL-On-Hadoop engine.
* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL.
This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`.
<table class="table-configuration">
<tr>
@ -62,17 +76,7 @@ Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You
</tr>
</table>
[<img align="right" src="http://img.youtube.com/vi/wqXXQhJ5Uk8/0.jpg" alt="zeppelin-view" hspace="10" width="250"></img>](https://www.youtube.com/watch?v=wqXXQhJ5Uk8)
This interpreter seamlessly supports the following SQL data processing engines:
* [PostgreSQL](http://www.postgresql.org/) - OSS, Object-relational database management system (ORDBMS)
* [Apache HAWQ](http://pivotal.io/big-data/pivotal-hawq) - Powerful [Open Source](https://wiki.apache.org/incubator/HAWQProposal) SQL-On-Hadoop engine.
* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL.
This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`.
### Create Interpreter
## Create Interpreter
By default Zeppelin creates one `PSQL` instance. You can remove it or create new instances.
Multiple PSQL instances can be created, each configured to the same or different backend databases. But over time a `Notebook` can have only one PSQL interpreter instance `bound`. That means you _cannot_ connect to different databases in the same `Notebook`. This is a known Zeppelin limitation.
@ -81,10 +85,10 @@ To create new PSQL instance open the `Interpreter` section and click the `+Creat
> Note: The `Name` of the instance is used only to distinct the instances while binding them to the `Notebook`. The `Name` is irrelevant inside the `Notebook`. In the `Notebook` you must use `%psql.sql` tag.
### Bind to Notebook
## Bind to Notebook
In the `Notebook` click on the `settings` icon in the top right corner. The select/deselect the interpreters to be bound with the `Notebook`.
### Configuration
## Configuration
You can modify the configuration of the PSQL from the `Interpreter` section. The PSQL interpreter expenses the following properties:
<table class="table-configuration">
@ -120,12 +124,12 @@ You can modify the configuration of the PSQL from the `Interpreter` section. Th
</tr>
</table>
### How to use
## How to use
```
Tip: Use (CTRL + .) for SQL auto-completion.
```
#### DDL and SQL commands
### DDL and SQL commands
Start the paragraphs with the full `%psql.sql` prefix tag! The short notation: `%psql` would still be able run the queries but the syntax highlighting and the auto-completions will be disabled.
You can use the standard CREATE / DROP / INSERT commands to create or modify the data model:
@ -154,7 +158,7 @@ select count(*) from mytable;
select * from mytable;
```
#### PSQL command line tools
### PSQL command line tools
Use the Shell Interpreter (`%sh`) to access the command line [PSQL](http://www.postgresql.org/docs/9.4/static/app-psql.html) interactively:
```bash
@ -179,7 +183,7 @@ This will produce output like this:
retail_demo | gpadmin
```
#### Apply Zeppelin Dynamic Forms
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
@ -191,7 +195,7 @@ ORDER BY count ${order=DESC,DESC|ASC}
LIMIT ${limit=10};
```
#### Example HAWQ PXF/HDFS Tables
### Example HAWQ PXF/HDFS Tables
Create HAWQ external table that read data from tab-separated-value data in HDFS.
```sql
@ -209,5 +213,5 @@ And retrieve content
select * from retail_demo.payment_methods_pxf
```
### Auto-completion
## Auto-completion
The PSQL Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. In addition to the SQL keyword the interpreter provides suggestions for the Schema, Table, Column names as well.

70
docs/interpreter/python.md
View file

@ -6,7 +6,9 @@ group: manual
---
{% include JB/setup %}
## Python 2 & 3 Interpreter for Apache Zeppelin
# Python 2 & 3 Interpreter for Apache Zeppelin
<div id="toc"></div>
## Configuration
<table class="table-configuration">
@ -16,12 +18,17 @@ group: manual
<th>Description</th>
</tr>
<tr>
<td>python</td>
<td>zeppelin.python</td>
<td>python</td>
<td>Path of the already installed Python binary (could be python2 or python3).
If python is not in your $PATH you can set the absolute directory (example : /usr/bin/python)
</td>
</tr>
<tr>
<td>zeppelin.python.maxResult</td>
<td>1000</td>
<td>Max number of dataframe rows to display.</td>
</tr>
</table>
## Enabling Python Interpreter
@ -39,7 +46,7 @@ To access the help, type **help()**
## Python modules
The interpreter can use all modules already installed (with pip, easy_install...)
## Apply Zeppelin Dynamic Forms
## Use Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your Python code.
**Zeppelin Dynamic Form can only be used if py4j Python library is installed in your system. If not, you can install it with `pip install py4j`.**
@ -58,9 +65,6 @@ print (z.select("f1",[("o1","1"),("o2","2")],"2"))
print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
```
## Zeppelin features not fully supported by the Python Interpreter
* Interrupt a paragraph execution (`cancel()` method) is currently only supported in Linux and MacOs. If interpreter runs in another operating system (for instance MS Windows) , interrupt a paragraph will close the whole interpreter. A JIRA ticket ([ZEPPELIN-893](https://issues.apache.org/jira/browse/ZEPPELIN-893)) is opened to implement this feature in a next release of the interpreter.
@ -68,50 +72,38 @@ print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
* Code-completion is currently not implemented.
## Matplotlib integration
The python interpreter can display matplotlib graph with the function **_zeppelin_show()_**
You need to already have matplotlib module installed and a running XServer to use this functionality !
The python interpreter can display matplotlib graph with the function `z.show()`.
You need to have matplotlib module installed and a XServer running to use this functionality !
```python
%python
import matplotlib.pyplot as plt
plt.figure()
(.. ..)
zeppelin_show(plt)
z.show(plt)
plt.close()
```
zeppelin_show function can take optional parameters to adapt graph width and height
z.show function can take optional parameters to adapt graph width and height
```python
%python
zeppelin_show(plt,width='50px')
zeppelin_show(plt,height='150px')
z.show(plt, width='50px')
z.show(plt, height='150px')
```
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/pythonMatplotlib.png" />
## Pandas integration
[Zeppelin Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table) provides simple API to visualize data in Pandas DataFrames, same as in Matplotlib.
Example:
```python
import pandas as pd
rates = pd.read_csv("bank.csv", sep=";")
z.show(rates)
```
[![pythonmatplotlib](../interpreter/screenshots/pythonMatplotlib.png)](/docs/interpreter/screenshots/pythonMatplotlib.png)
## Technical description
## Technical description - Interpreter architecture
### Dev prerequisites
* Python 2 and 3 installed with py4j (0.9.2) and matplotlib (1.31 or later) installed on each
* Tests only checks the interpreter logic and starts any Python process ! Python process is mocked with a class that simply output it input.
* Make sure the code wrote in bootstrap.py and bootstrap_input.py is Python2 and 3 compliant.
* Use PEP8 convention for python code.
### Technical overview
* When interpreter is starting it launches a python process inside a Java ProcessBuilder. Python is started with -i (interactive mode) and -u (unbuffered stdin, stdout and stderr) options. Thus the interpreter has a "sleeping" python process.
* Interpreter sends command to python with a Java `outputStreamWiter` and read from an `InputStreamReader`. To know when stop reading stdout, interpreter sends `print "*!?flush reader!?*"`after each command and reads stdout until he receives back the `*!?flush reader!?*`.
* When interpreter is starting, it sends some Python code (bootstrap.py and bootstrap_input.py) to initialize default behavior and functions (`help(), z.input()...`). bootstrap_input.py is sent only if py4j library is detected inside Python process.
* [Py4J](https://www.py4j.org/) python and java libraries is used to load Input zeppelin Java class into the python process (make java code with python code !). Therefore the interpreter can directly create Zeppelin input form inside the Python process (and eventually with some python variable already defined). JVM opens a random open port to be accessible from python process.
* JavaBuilder can't send SIGINT signal to interrupt paragraph execution. Therefore interpreter directly send a `kill SIGINT PID` to python process to interrupt execution. Python process catch SIGINT signal with some code defined in bootstrap.py
* Matplotlib display feature is made with SVG export (in string) and then displays it with html code.
For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md).

176
docs/interpreter/r.md
View file

@ -6,104 +6,11 @@ group: manual
---
{% include JB/setup %}
## R Interpreter
# R Interpreter for Apache Zeppelin
This is a the Apache Zeppelin project, with the addition of support for the R programming language and R-spark integration.
<div id="toc"></div>
### Requirements
Additional requirements for the R interpreter are:
* R 3.1 or later (earlier versions may work, but have not been tested)
* The `evaluate` R package.
For full R support, you will also need the following R packages:
* `knitr`
* `repr` -- available with `devtools::install_github("IRkernel/repr")`
* `htmltools` -- required for some interactive plotting
* `base64enc` -- required to view R base plots
### Configuration
To run Zeppelin with the R Interpreter, the SPARK_HOME environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`.
If it is not set, the R Interpreter will not be able to interface with Spark.
You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up.
### Using the R Interpreter
By default, the R Interpreter appears as two Zeppelin Interpreters, `%r` and `%knitr`.
`%r` will behave like an ordinary REPL. You can execute commands as in the CLI.
[![2+2](screenshots/repl2plus2.png)](screenshots/repl2plus2.png)
R base plotting is fully supported
[![replhist](screenshots/replhist.png)](screenshots/replhist.png)
If you return a data.frame, Zeppelin will attempt to display it using Zeppelin's built-in visualizations.
[![replhist](screenshots/replhead.png)](screenshots/replhead.png)
`%knitr` interfaces directly against `knitr`, with chunk options on the first line:
[![knitgeo](screenshots/knitgeo.png)](screenshots/knitgeo.png)
[![knitstock](screenshots/knitstock.png)](screenshots/knitstock.png)
[![knitmotion](screenshots/knitmotion.png)](screenshots/knitmotion.png)
The two interpreters share the same environment. If you define a variable from `%r`, it will be within-scope if you then make a call using `knitr`.
### Using SparkR & Moving Between Languages
If `SPARK_HOME` is set, the `SparkR` package will be loaded automatically:
[![sparkrfaithful](screenshots/sparkrfaithful.png)](screenshots/sparkrfaithful.png)
The Spark Context and SQL Context are created and injected into the local environment automatically as `sc` and `sql`.
The same context are shared with the `%spark`, `%sql` and `%pyspark` interpreters:
[![backtoscala](screenshots/backtoscala.png)](screenshots/backtoscala.png)
You can also make an ordinary R variable accessible in scala and Python:
[![varr1](screenshots/varr1.png)](screenshots/varr1.png)
And vice versa:
[![varscala](screenshots/varscala.png)](screenshots/varscala.png)
[![varr2](screenshots/varr2.png)](screenshots/varr2.png)
### Caveats & Troubleshooting
* Almost all issues with the R interpreter turned out to be caused by an incorrectly set `SPARK_HOME`. The R interpreter must load a version of the `SparkR` package that matches the running version of Spark, and it does this by searching `SPARK_HOME`. If Zeppelin isn't configured to interface with Spark in `SPARK_HOME`, the R interpreter will not be able to connect to Spark.
* The `knitr` environment is persistent. If you run a chunk from Zeppelin that changes a variable, then run the same chunk again, the variable has already been changed. Use immutable variables.
* (Note that `%spark.r` and `$r` are two different ways of calling the same interpreter, as are `%spark.knitr` and `%knitr`. By default, Zeppelin puts the R interpreters in the `%spark.` Interpreter Group.
* Using the `%r` interpreter, if you return a data.frame, HTML, or an image, it will dominate the result. So if you execute three commands, and one is `hist()`, all you will see is the histogram, not the results of the other commands. This is a Zeppelin limitation.
* If you return a data.frame (for instance, from calling `head()`) from the `%spark.r` interpreter, it will be parsed by Zeppelin's built-in data visualization system.
* Why `knitr` Instead of `rmarkdown`? Why no `htmlwidgets`? In order to support `htmlwidgets`, which has indirect dependencies, `rmarkdown` uses `pandoc`, which requires writing to and reading from disc. This makes it many times slower than `knitr`, which can operate entirely in RAM.
* Why no `ggvis` or `shiny`? Supporting `shiny` would require integrating a reverse-proxy into Zeppelin, which is a task.
* Max OS X & case-insensitive filesystem. If you try to install on a case-insensitive filesystem, which is the Mac OS X default, maven can unintentionally delete the install directory because `r` and `R` become the same subdirectory.
* Error `unable to start device X11` with the repl interpreter. Check your shell login scripts to see if they are adjusting the `DISPLAY` environment variable. This is common on some operating systems as a workaround for ssh issues, but can interfere with R plotting.
* akka Library Version or `TTransport` errors. This can happen if you try to run Zeppelin with a SPARK_HOME that has a version of Spark other than the one specified with `-Pspark-1.x` when Zeppelin was compiled.
## R Interpreter for Apache Zeppelin
## Overview
[R](https://www.r-project.org) is a free software environment for statistical computing and graphics.
@ -135,3 +42,80 @@ We recommend you to also install the following optional R libraries for happy da
+ caret
+ sqldf
+ wordcloud
## Configuration
To run Zeppelin with the R Interpreter, the `SPARK_HOME` environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`.
If it is not set, the R Interpreter will not be able to interface with Spark.
You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up.
## Using the R Interpreter
By default, the R Interpreter appears as two Zeppelin Interpreters, `%r` and `%knitr`.
`%r` will behave like an ordinary REPL. You can execute commands as in the CLI.
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/repl2plus2.png" width="700px"/>
R base plotting is fully supported
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/replhist.png" width="550px"/>
If you return a data.frame, Zeppelin will attempt to display it using Zeppelin's built-in visualizations.
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/replhead.png" width="550px"/>
`%knitr` interfaces directly against `knitr`, with chunk options on the first line:
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/knitgeo.png" width="550px"/>
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/knitstock.png" width="550px"/>
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/knitmotion.png" width="550px"/>
The two interpreters share the same environment. If you define a variable from `%r`, it will be within-scope if you then make a call using `knitr`.
## Using SparkR & Moving Between Languages
If `SPARK_HOME` is set, the `SparkR` package will be loaded automatically:
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/sparkrfaithful.png" width="550px"/>
The Spark Context and SQL Context are created and injected into the local environment automatically as `sc` and `sql`.
The same context are shared with the `%spark`, `%sql` and `%pyspark` interpreters:
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/backtoscala.png" width="700px"/>
You can also make an ordinary R variable accessible in scala and Python:
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/varr1.png" width="550px"/>
And vice versa:
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/varscala.png" width="550px"/>
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/varr2.png" width="550px"/>
## Caveats & Troubleshooting
* Almost all issues with the R interpreter turned out to be caused by an incorrectly set `SPARK_HOME`. The R interpreter must load a version of the `SparkR` package that matches the running version of Spark, and it does this by searching `SPARK_HOME`. If Zeppelin isn't configured to interface with Spark in `SPARK_HOME`, the R interpreter will not be able to connect to Spark.
* The `knitr` environment is persistent. If you run a chunk from Zeppelin that changes a variable, then run the same chunk again, the variable has already been changed. Use immutable variables.
* (Note that `%spark.r` and `$r` are two different ways of calling the same interpreter, as are `%spark.knitr` and `%knitr`. By default, Zeppelin puts the R interpreters in the `%spark.` Interpreter Group.
* Using the `%r` interpreter, if you return a data.frame, HTML, or an image, it will dominate the result. So if you execute three commands, and one is `hist()`, all you will see is the histogram, not the results of the other commands. This is a Zeppelin limitation.
* If you return a data.frame (for instance, from calling `head()`) from the `%spark.r` interpreter, it will be parsed by Zeppelin's built-in data visualization system.
* Why `knitr` Instead of `rmarkdown`? Why no `htmlwidgets`? In order to support `htmlwidgets`, which has indirect dependencies, `rmarkdown` uses `pandoc`, which requires writing to and reading from disc. This makes it many times slower than `knitr`, which can operate entirely in RAM.
* Why no `ggvis` or `shiny`? Supporting `shiny` would require integrating a reverse-proxy into Zeppelin, which is a task.
* Max OS X & case-insensitive filesystem. If you try to install on a case-insensitive filesystem, which is the Mac OS X default, maven can unintentionally delete the install directory because `r` and `R` become the same subdirectory.
* Error `unable to start device X11` with the repl interpreter. Check your shell login scripts to see if they are adjusting the `DISPLAY` environment variable. This is common on some operating systems as a workaround for ssh issues, but can interfere with R plotting.
* akka Library Version or `TTransport` errors. This can happen if you try to run Zeppelin with a SPARK_HOME that has a version of Spark other than the one specified with `-Pspark-1.x` when Zeppelin was compiled.

19
docs/interpreter/scalding.md
View file

@ -6,17 +6,20 @@ group: manual
---
{% include JB/setup %}
## Scalding Interpreter for Apache Zeppelin
# Scalding Interpreter for Apache Zeppelin
<div id="toc"></div>
[Scalding](https://github.com/twitter/scalding) is an open source Scala library for writing MapReduce jobs.
### Building the Scalding Interpreter
## Building the Scalding Interpreter
You have to first build the Scalding interpreter by enable the **scalding** profile as follows:
```
mvn clean package -Pscalding -DskipTests
```
### Enabling the Scalding Interpreter
## Enabling the Scalding Interpreter
In a notebook, to enable the **Scalding** interpreter, click on the **Gear** icon,select **Scalding**, and hit **Save**.
<center>
@ -27,7 +30,7 @@ In a notebook, to enable the **Scalding** interpreter, click on the **Gear** ico
</center>
### Configuring the Interpreter
## Configuring the Interpreter
Scalding interpreter runs in two modes:
@ -65,9 +68,9 @@ For reducer estimation, you need to add something like:
If you want to control the maximum number of open interpreters, you have to select "scoped" interpreter for note
option and set max.open.instances argument.
### Testing the Interpreter
## Testing the Interpreter
#### Local mode
### Local mode
In example, by using the [Alice in Wonderland](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) tutorial,
we will count words (of course!), and plot a graph of the top 10 words in the book.
@ -111,7 +114,7 @@ If you click on the icon for the pie chart, you should be able to see a chart li
![Scalding - Pie - Chart](../assets/themes/zeppelin/img/docs-img/scalding-pie.png)
#### HDFS mode
### HDFS mode
**Test mode**
@ -146,7 +149,7 @@ a.toList
This command should create a map reduce job.
### Future Work
## Future Work
* Better user feedback (hadoop url, progress updates)
* Ability to cancel jobs
* Ability to dynamically load jars without restarting the interpreter

29
docs/interpreter/spark.md
View file

@ -7,8 +7,14 @@ group: manual
{% include JB/setup %}
## Spark Interpreter for Apache Zeppelin
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
# Spark Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Apache Spark](http://spark.apache.org) is a fast and general-purpose cluster computing system.
It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs
Apache Spark is supported in Zeppelin with
Spark Interpreter group, which consists of five interpreters.
<table class="table-configuration">
@ -45,7 +51,7 @@ Spark Interpreter group, which consists of five interpreters.
</table>
## Configuration
The Spark interpreter can be configured with properties provided by Zeppelin.
The Spark interpreter can be configured with properties provided by Zeppelin.
You can also set other Spark properties which are not listed in the table. For a list of additional properties, refer to [Spark Available Properties](http://spark.apache.org/docs/latest/configuration.html#available-properties).
<table class="table-configuration">
<tr>
@ -111,6 +117,11 @@ You can also set other Spark properties which are not listed in the table. For a
<td>true</td>
<td>Use HiveContext instead of SQLContext if it is true.</td>
</tr>
<tr>
<td>zeppelin.spark.importImplicit</td>
<td>true</td>
<td>Import implicits, UDF collection, and sql if set true.</td>
</tr>
</table>
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
@ -195,13 +206,13 @@ Here are few examples:
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg"
* SPARK_HOME/conf/spark-defaults.conf
spark.jars /path/mylib1.jar,/path/mylib2.jar
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
spark.jars /path/mylib1.jar,/path/mylib2.jar
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
### 3. Dynamic Dependency Loading via %dep interpreter
> Note: `%dep` interpreter is deprecated since v0.6.0.
@ -339,7 +350,7 @@ select * from ${table=defaultTableName} where text like '%${search}%'
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
### Interpreter setting option.
## Interpreter setting option
Interpreter setting can choose one of 'shared', 'scoped', 'isolated' option. Spark interpreter creates separate scala compiler per each notebook but share a single SparkContext in 'scoped' mode (experimental). It creates separate SparkContext per each notebook in 'isolated' mode.
@ -349,7 +360,7 @@ Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark o
<img src="../assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png">
####Configuration Setup
### Configuration Setup
1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf.
This is to make the server communicate with KDC.

22
docs/manual/dynamicform.md
View file

@ -19,16 +19,18 @@ limitations under the License.
-->
{% include JB/setup %}
## Dynamic Form
# Dynamic Form
Zeppelin dynamically creates input forms. Depending on language backend, there're two different ways to create dynamic form.
<div id="toc"></div>
Apache Zeppelin dynamically creates input forms. Depending on language backend, there're two different ways to create dynamic form.
Custom language backend can select which type of form creation it wants to use.
### Using form Templates
## Using form Templates
This mode creates form using simple template language. It's simple and easy to use. For example Markdown, Shell, SparkSql language backend uses it.
#### Text input form
### Text input form
To create text input form, use `${formName}` templates.
@ -42,7 +44,7 @@ Also you can provide default value, using `${formName=defaultValue}`.
<img src="../assets/themes/zeppelin/img/screenshots/form_input_default.png" />
#### Select form
### Select form
To create select form, use `${formName=defaultValue,option1|option2...}`
@ -54,7 +56,7 @@ Also you can separate option's display name and value, using `${formName=default
<img src="../assets/themes/zeppelin/img/screenshots/form_select_displayname.png" />
#### Checkbox form
### Checkbox form
For multi-selection, you can create a checkbox form using `${checkbox:formName=defaultValue1|defaultValue2...,option1|option2...}`. The variable will be substituted by a comma-separated string based on the selected items. For example:
@ -64,13 +66,13 @@ Besides, you can specify the delimiter using `${checkbox(delimiter):formName=...
<img src="../assets/themes/zeppelin/img/screenshots/form_checkbox_delimiter.png">
### Creates Programmatically
## Creates Programmatically
Some language backend uses programmatic way to create form. For example [ZeppelinContext](../interpreter/spark.html#zeppelincontext) provides form creation API
Here're some examples.
####Text input form
### Text input form
<div class="codetabs">
<div data-lang="scala" markdown="1">
@ -91,7 +93,7 @@ print("Hello "+z.input("name"))
</div>
<img src="../assets/themes/zeppelin/img/screenshots/form_input_prog.png" />
####Text input form with default value
### Text input form with default value
<div class="codetabs">
<div data-lang="scala" markdown="1">
@ -112,7 +114,7 @@ print("Hello "+z.input("name", "sun"))
</div>
<img src="../assets/themes/zeppelin/img/screenshots/form_input_default_prog.png" />
####Select form
### Select form
<div class="codetabs">
<div data-lang="scala" markdown="1">

25
docs/manual/dynamicinterpreterload.md
View file

@ -19,12 +19,13 @@ limitations under the License.
-->
{% include JB/setup %}
## Dynamic Interpreter Loading using REST API
# Dynamic Interpreter Loading using REST API
<div id="toc"></div>
Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal).
Before we start, if you are not familiar with the concept of **Zeppelin interpreter**, you can check out [Overview of Zeppelin interpreter](../manual/interpreters.html) first.
<br/>
## Overview
In the past, Zeppelin was loading interpreter binaries from `/interpreter/[interpreter_name]` directory. They were configured by `zeppelin.interpreters` property in `conf/zeppelin-site.xml` or `ZEPPELIN_INTERPRETERS` env variables in `conf/zeppelin-env.sh`. They were loaded on Zeppelin server startup and stayed alive until the server was stopped.
In order to simplify using 3rd party interpreters, we changed this way to **dynamically** load interpreters from **Maven Repository** using **REST API**. Hopefully, the picture below will help you to understand the process.
@ -32,7 +33,7 @@ In order to simplify using 3rd party interpreters, we changed this way to **dyna
## Load & Unload Interpreters Using REST API
### 1. Load
### Load
You can **load** interpreters located in Maven repository using REST API, like this:
( Maybe, you are unfamiliar with `[interpreter_group_name]` or `[interpreter_name]`. If so, please checkout [Interpreters in Zeppelin](../manual/interpreter.html) again. )
@ -69,21 +70,21 @@ http://127.0.0.1:8080/api/interpreter/load/md/markdown
The meaning of each parameters is:
1. **Artifact**
- groupId: org.apache.zeppelin
- artifactId: zeppelin-markdown
- version: 0.6.0-SNAPSHOT
- groupId: org.apache.zeppelin
- artifactId: zeppelin-markdown
- version: 0.6.0-SNAPSHOT
2. **Class Name**
- Package Name: org.apache.zeppelin
- Interpreter Class Name: markdown.Markdown
- Package Name: org.apache.zeppelin
- Interpreter Class Name: markdown.Markdown
3. **Repository ( optional )**
- Url: http://dl.bintray.com/spark-packages/maven
- Snapshot: false
- Url: http://dl.bintray.com/spark-packages/maven
- Snapshot: false
> <b>Please note: </b>The interpreters you downloaded need to be **reload**, when your Zeppelin server is down.
### 2. Unload
### Unload
If you want to **unload** the interpreters using REST API,
```
@ -95,7 +96,7 @@ In this case, the Restful method will be <code>**DELETE**</code>.
## What is the next step after Loading ?
### Q1. Where is the location of interpreters you downloaded ?
Actually, the answer about this question is in the above picture. Once the REST API is called, the `.jar` files of interpreters you get are saved under `ZEPPELIN_HOME/local-repo` first. Then, they will be copied to `ZEPPELIN_HOME/interpreter` directory. So, please checkout your `ZEPPELIN_HOME/interpreter`.
### Q2. Then, how can I use this interpreter ?

166
docs/manual/interpreterinstallation.md Normal file
View file

@ -0,0 +1,166 @@
---
layout: page
title: "Interpreter Installation"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Interpreter Installation
<div id="toc"></div>
Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters.
## Community managed interpreters
Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters).
If you downloaded `netinst` binary package, you need to install by using below commands.
#### Install all community managed interpreters
```
./bin/install-interpreter.sh --all
```
#### Install specific interpreters
```
./bin/install-interpreter.sh --name md,shell,jdbc,python
```
You can get full list of community managed interpreters by running
```
./bin/install-interpreter.sh --list
```
Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
## 3rd party interpreters
You can also install 3rd party interpreters located in the maven repository by using below commands.
#### Install 3rd party interpreters
```
./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1
```
The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration).
And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
#### Install multiple 3rd party interpreters at once
```
./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2
```
`--name` and `--artifact` arguments will recieve comma separated list.
## Available community managed interpreters
You can also find the below community managed interpreter list in `conf/interpreter-list` file.
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Maven Artifact</th>
<th>Description</th>
</tr>
<tr>
<td>alluxio</td>
<td>org.apache.zeppelin:zeppelin-alluxio:0.6.0</td>
<td>Alluxio interpreter</td>
</tr>
<tr>
<td>angular</td>
<td>org.apache.zeppelin:zeppelin-angular:0.6.0</td>
<td>HTML and AngularJS view rendering</td>
</tr>
<tr>
<td>cassandra</td>
<td>org.apache.zeppelin:zeppelin-cassandra:0.6.0</td>
<td>Cassandra interpreter</td>
</tr>
<tr>
<td>elasticsearch</td>
<td>org.apache.zeppelin:zeppelin-elasticsearch:0.6.0</td>
<td>Elasticsearch interpreter</td>
</tr>
<tr>
<td>file</td>
<td>org.apache.zeppelin:zeppelin-file:0.6.0</td>
<td>HDFS file interpreter</td>
</tr>
<tr>
<td>flink</td>
<td>org.apache.zeppelin:zeppelin-flink:0.6.0</td>
<td>Flink interpreter</td>
</tr>
<tr>
<td>hbase</td>
<td>org.apache.zeppelin:zeppelin-hbase:0.6.0</td>
<td>Hbase interpreter</td>
</tr>
<tr>
<td>ignite</td>
<td>org.apache.zeppelin:zeppelin-ignite:0.6.0</td>
<td>Ignite interpreter</td>
</tr>
<tr>
<td>jdbc</td>
<td>org.apache.zeppelin:zeppelin-jdbc:0.6.0</td>
<td>Jdbc interpreter</td>
</tr>
<tr>
<td>kylin</td>
<td>org.apache.zeppelin:zeppelin-kylin:0.6.0</td>
<td>Kylin interpreter</td>
</tr>
<tr>
<td>lens</td>
<td>org.apache.zeppelin:zeppelin-lens:0.6.0</td>
<td>Lens interpreter</td>
</tr>
<tr>
<td>livy</td>
<td>org.apache.zeppelin:zeppelin-livy:0.6.0</td>
<td>Livy interpreter</td>
</tr>
<tr>
<td>md</td>
<td>org.apache.zeppelin:zeppelin-markdown:0.6.0</td>
<td>Markdown support</td>
</tr>
<tr>
<td>postgresql</td>
<td>org.apache.zeppelin:zeppelin-postgresql:0.6.0</td>
<td>Postgresql interpreter</td>
</tr>
<tr>
<td>python</td>
<td>org.apache.zeppelin:zeppelin-python:0.6.0</td>
<td>Python interpreter</td>
</tr>
<tr>
<td>shell</td>
<td>org.apache.zeppelin:zeppelin-shell:0.6.0</td>
<td>Shell command</td>
</tr>
</table>

22
docs/manual/interpreters.md
View file

@ -19,7 +19,12 @@ limitations under the License.
-->
{% include JB/setup %}
## Interpreters in Zeppelin
# Interpreters in Apache Zeppelin
<div id="toc"></div>
## Overview
In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin.
The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin.
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, JDBC, Markdown, Shell and so on.
@ -29,12 +34,12 @@ Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_create.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_create.png" width="280px">
## What is Zeppelin Interpreter Setting?
## What is interpreter setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting.png" width="500px">
Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
@ -44,14 +49,15 @@ Each notebook can be bound to multiple Interpreter Settings using setting icon o
## What is Zeppelin Interpreter Group?
## What is interpreter group?
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, SparkSQL and the dependency loader.
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png" width="500px">
## Interpreter binding mode
@ -62,7 +68,7 @@ In 'shared' mode, every notebook bound to the Interpreter Setting will share the
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">
## Connecting to the Existing Remote Interpreter
## Connecting to the existing remote interpreter
Zeppelin users can start interpreter thread embedded in their service. This will provide flexibility to user to start interpreter on remote host. To start interpreter along with your service you have to create an instance of ``RemoteInterpreterServer`` and start it as follows:
@ -75,4 +81,4 @@ interpreter.start()
The above code will start interpreter thread inside your process. Once the interpreter is started you can configure zeppelin to connect to RemoteInterpreter by checking **Connect to existing process** checkbox and then provide **Host** and **Port** on which interpreter porocess is listening as shown in the image below:
<img src="../assets/themes/zeppelin/img/screenshots/existing_interpreter.png" width="400px">
<img src="../assets/themes/zeppelin/img/screenshots/existing_interpreter.png" width="450px">

131
docs/manual/notebookashomepage.md
View file

@ -19,91 +19,84 @@ limitations under the License.
-->
{% include JB/setup %}
## Customize your zeppelin homepage
Zeppelin allows you to use one of the notebooks you create as your zeppelin Homepage.
With that you can brand your zeppelin installation,
adjust the instruction to your users needs and even translate to other languages.
# Customize Apache Zeppelin homepage
<br />
### How to set a notebook as your zeppelin homepage
<div id="toc"></div>
Apache Zeppelin allows you to use one of the notebooks you create as your Zeppelin Homepage.
With that you can brand your Zeppelin installation, adjust the instruction to your users needs and even translate to other languages.
## How to set a notebook as your Zeppelin homepage
The process for creating your homepage is very simple as shown below:
1. Create a notebook using zeppelin
2. Set the notebook id in the config file
3. Restart zeppelin
1. Create a notebook using Zeppelin
2. Set the notebook id in the config file
3. Restart Zeppelin
<br />
#### Create a notebook using zeppelin
Create a new notebook using zeppelin,
you can use ```%md``` interpreter for markdown content or any other interpreter you like.
### Create a notebook using Zeppelin
Create a new notebook using Zeppelin,
you can use ```%md``` interpreter for markdown content or any other interpreter you like.
You can also use the display system to generate [text](../displaysystem/basicdisplaysystem.html#text), [html](../displaysystem/basicdisplaysystem.html#html), [table](../displaysystem/basicdisplaysystem.html#table) or
Angular ([backend API](../displaysystem/back-end-angular.html), [frontend API](../displaysystem/front-end-angular.html)).
You can also use the display system to generate [text](../displaysystem/display.html),
[html](../displaysystem/display.html#html),[table](../displaysystem/table.html) or
[angular](../displaysystem/angular.html)
Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide
the code sections.
Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide
the code sections.
### Set the notebook id in the config file
To set the notebook id in the config file, you should copy it from the last word in the notebook url.
For example,
<br />
#### Set the notebook id in the config file
To set the notebook id in the config file you should copy it from the last word in the notebook url
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_id.png" />
for example
Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable
or ```zeppelin.notebook.homescreen``` property.
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_id.png" />
You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable
or ```zeppelin.notebook.homescreen.hide``` property to hide the new notebook from the notebook list.
Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable
or ```zeppelin.notebook.homescreen``` property.
You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable
or ```zeppelin.notebook.homescreen.hide``` property to hide the new notebook from the notebook list.
<br />
#### Restart zeppelin
Restart your zeppelin server
```
./bin/zeppelin-deamon stop
./bin/zeppelin-deamon start
```
####That's it! Open your browser and navigate to zeppelin and see your customized homepage...
### Restart Zeppelin
Restart your Zeppelin server
```
./bin/zeppelin-deamon stop
./bin/zeppelin-deamon start
```
That's it! Open your browser and navigate to Apache Zeppelin and see your customized homepage.
<br />
### Show notebooks list in your custom homepage
If you want to display the list of notebooks on your custom zeppelin homepage all
## Show notebooks list in your custom homepage
If you want to display the list of notebooks on your custom Apache Zeppelin homepage all
you need to do is use our %angular support.
<br />
Add the following code to a paragraph in you home page and run it... walla! you have your notebooks list.
Add the following code to a paragraph in you home page and run it... walla! you have your notebooks list.
```javascript
println(
"""%angular
<div class="col-md-4" ng-controller="HomeCtrl as home">
<h4>Notebooks</h4>
<div>
<h5><a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;">
<i style="font-size: 15px;" class="icon-notebook"></i> Create new note</a></h5>
<ul style="list-style-type: none;">
<li ng-repeat="note in home.notes.list track by $index"><i style="font-size: 10px;" class="icon-doc"></i>
<a style="text-decoration: none;" href="#/notebook/{{note.id}}">{{noteName(note)}}</a>
</li>
</ul>
</div>
</div>
""")
```
After running the notebook you will see output similar to this one:
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_list.png" />
The main trick here relays in linking the ```<div>``` to the controller:
```javascript
```javascript
println(
"""%angular
<div class="col-md-4" ng-controller="HomeCtrl as home">
```
<h4>Notebooks</h4>
<div>
<h5><a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;">
<i style="font-size: 15px;" class="icon-notebook"></i> Create new note</a></h5>
<ul style="list-style-type: none;">
<li ng-repeat="note in home.notes.list track by $index"><i style="font-size: 10px;" class="icon-doc"></i>
<a style="text-decoration: none;" href="#/notebook/{{note.id}}">{{noteName(note)}}</a>
</li>
</ul>
</div>
</div>
""")
```
Once we have ```home``` as our controller variable in our ```<div></div>```
we can use ```home.notes.list``` to get access to the notebook list.
After running the notebook you will see output similar to this one:
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_list.png" />
The main trick here relays in linking the ```<div>``` to the controller:
```javascript
<div class="col-md-4" ng-controller="HomeCtrl as home">
```
Once we have ```home``` as our controller variable in our ```<div></div>```
we can use ```home.notes.list``` to get access to the notebook list.

15
docs/manual/publish.md
View file

@ -19,13 +19,14 @@ limitations under the License.
-->
{% include JB/setup %}
## How can you publish your paragraph ?
Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website.
# How can you publish your paragraph ?
<div id="toc"></div>
Apache Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website.
It's very straightforward. Just use `<iframe>` tag in your page.
> **Warning**: Please use this feature with caution and in a trusted environment only, as Zeppelin entire Webapp could be accessible for whoever visits your website.
### Copy a Paragraph Link
## Copy a Paragraph Link
A first step to publish your paragraph result is **Copy a Paragraph Link**.
* After running a paragraph in your Zeppelin notebook, click a gear button located on the right side. Then, click **Link this Paragraph** menu like below image.
@ -34,7 +35,7 @@ A first step to publish your paragraph result is **Copy a Paragraph Link**.
* Just copy the provided link.
<center><img src="../assets/themes/zeppelin/img/docs-img/copy-the-link.png" height="100%" width="100%"></center>
### Embed the Paragraph to Your Website
## Embed the Paragraph to Your Website
For publishing the copied paragraph, you may use `<iframe>` tag in your website page.
For example,
@ -45,4 +46,4 @@ For example,
Finally, you can show off your beautiful visualization results in your website.
<center><img src="../assets/themes/zeppelin/img/docs-img/your-website.png" height="90%" width="90%"></center>
> **Note**: To embed the paragraph in a website, Zeppelin needs to be reachable by that website.
> **Note**: To embed the paragraph in a website, Apache Zeppelin needs to be reachable by that website. And please use this feature with caution and in a trusted environment only, as Zeppelin entire Webapp could be accessible by whoever visits your website.

26
docs/quickstart/explorezeppelinui.md
View file

@ -1,6 +1,6 @@
---
layout: page
title: "Explore Zeppelin UI"
title: "Explore Apache Zeppelin UI"
description: "Description of Zeppelin UI Layout"
group: quickstart
---
@ -17,9 +17,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
## Home Page
# Explore Apache Zeppelin UI
The first time you connect to Zeppelin, you'll land at the main page similar to the below screen capture
<div id="toc"></div>
## Main home
The first time you connect to Zeppelin, you'll land at the main page similar to the below screen capture.
<img src="../assets/themes/zeppelin/img/ui-img/homepage.png" />
@ -28,7 +32,7 @@ On the left of the page are listed all existing notes. Those notes are stored by
You can filter them by name using the input text form. You can also create an new note, refresh the list of existing notes
(in case you manually copy them into the `$ZEPPELIN_HOME/notebook` folder) and import a note.
<img src="../assets/themes/zeppelin/img/ui-img/notes_management.png" />
<img src="../assets/themes/zeppelin/img/ui-img/notes_management.png" width="230px" />
When clicking on `Import Note` link, a new dialog open. From there you can import your note from local disk or from a remote location
if you provide the URL.
@ -40,7 +44,7 @@ By default, the name of the imported note is the same as the original note but y
<br />
## Menus
### 1. Notebook
### Notebook
The `Notebook` menu proposes almost the same features as the note management section in the home page. From the drop-down menu you can:
@ -48,9 +52,9 @@ The `Notebook` menu proposes almost the same features as the note management sec
2. Filter node by name
3. Create a new note
<img src="../assets/themes/zeppelin/img/ui-img/notebook_menu.png" />
<img src="../assets/themes/zeppelin/img/ui-img/notebook_menu.png" width="170px" />
### 2. Interpreter
### Interpreter
In this menu you can:
@ -59,7 +63,7 @@ In this menu you can:
<img src="../assets/themes/zeppelin/img/ui-img/interpreter_menu.png" />
### 3. Configuration
### Configuration
This menu displays all the Zeppelin configuration that are set in the config file `$ZEPPELIN_HOME/conf/zeppelin-site.xml`
@ -88,7 +92,7 @@ On the top-right corner of each paragraph there are some commands to:
To configure the paragraph, just click on the gear icon:
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_configuration_dialog.png" />
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_configuration_dialog.png" width="180px" />
From this dialog, you can (in descending order):
@ -124,7 +128,7 @@ In the middle of the toolbar you can find the command buttons:
* delete the note
* schedule the execution of **all paragraph** using a CRON syntax
<img src="../assets/themes/zeppelin/img/ui-img/note_commands.png" />
<img src="../assets/themes/zeppelin/img/ui-img/note_commands.png" width="300px"/>
On the right of the note tool bar you can find configuration icons:
@ -133,4 +137,4 @@ On the right of the note tool bar you can find configuration icons:
* configure the note permissions
* switch the node display mode between `default`, `simple` and `report`
<img src="../assets/themes/zeppelin/img/ui-img/note_configuration.png" />
<img src="../assets/themes/zeppelin/img/ui-img/note_configuration.png" width="180px"/>

15
docs/quickstart/tutorial.md
View file

@ -17,16 +17,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
## Zeppelin Tutorial
# Zeppelin Tutorial
<div id="toc"></div>
This tutorial walks you through some of the fundamental Zeppelin concepts. We will assume you have already installed Zeppelin. If not, please see [here](../install/install.html) first.
Current main backend processing engine of Zeppelin is [Apache Spark](https://spark.apache.org). If you're new to this system, you might want to start by getting an idea of how it processes data to get the most out of Zeppelin.
<br />
## Tutorial with Local File
### 1. Data Refine
### Data Refine
Before you start Zeppelin tutorial, you will need to download [bank.zip](http://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip).
@ -52,7 +53,7 @@ val bank = bankText.map(s=>s.split(";")).filter(s=>s(0)!="\"age\"").map(
bank.toDF().registerTempTable("bank")
```
### 2. Data Retrieval
### Data Retrieval
Suppose we want to see age distribution from `bank`. To do this, run:
@ -75,7 +76,7 @@ Now we want to see age distribution with certain marital status and add combo bo
<br />
## Tutorial with Streaming Data
### 1. Data Refine
### Data Refine
Since this tutorial is based on Twitter's sample tweet stream, you must configure authentication with a Twitter account. To do this, take a look at [Twitter Credential Setup](https://databricks-training.s3.amazonaws.com/realtime-processing-with-spark-streaming.html#twitter-credential-setup). After you get API keys, you should fill out credential related values(`apiKey`, `apiSecret`, `accessToken`, `accessTokenSecret`) with your API keys on following script.
@ -135,7 +136,7 @@ twt.print
ssc.start()
```
### 2. Data Retrieval
### Data Retrieval
For each following script, every time you click run button you will see different result since it is based on real-time data.
@ -192,4 +193,4 @@ To check how people think about girls using `sentiment` function we've made abov
```sql
%sql select sentiment(text), count(1) from tweets where text like '%girl%' group by sentiment(text)
```
```

21
docs/rest-api/rest-configuration.md
View file

@ -19,18 +19,22 @@ limitations under the License.
-->
{% include JB/setup %}
## Zeppelin REST API
Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
# Apache Zeppelin Configuration REST API
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`. Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
<div id="toc"></div>
If you work with Zeppelin and find a need for an additional REST API, please [file an issue or send us mail](../../community.html).
## Overview
Apache Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html).
nd a need for an additional REST API, please [file an issue or send us mail](../../community.html).
<br />
## Configuration REST API list
### List Configurations
### List all key/value pair of configurations
<table class="table-configuration">
<col width="200">
<tr>
@ -73,7 +77,7 @@ limitations under the License.
"zeppelin.server.context.path": "/",
"zeppelin.ssl.keystore.type": "JKS",
"zeppelin.ssl.truststore.path": "truststore",
"zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter",
"zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter",
"zeppelin.ssl": "false",
"zeppelin.notebook.autoInterpreterBinding": "true",
"zeppelin.notebook.homescreen": "",
@ -89,7 +93,7 @@ limitations under the License.
<br/>
### List Configurations(prefix match)
### List all prefix matched key/value pair of configurations
<table class="table-configuration">
<col width="200">
<tr>
@ -128,3 +132,4 @@ limitations under the License.
</td>
</tr>
</table>

35
docs/rest-api/rest-interpreter.md
View file

@ -19,19 +19,22 @@ limitations under the License.
-->
{% include JB/setup %}
## Zeppelin REST API
Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
# Apache Zeppelin Interpreter REST API
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`. Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
<div id="toc"></div>
If you work with Zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.apache.org/community.html).
## Overview
Apache Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html).
<br />
## Interpreter REST API List
The role of registered interpreters, settings and interpreters group are described in [here](../manual/interpreters.html).
The role of registered interpreters, settings and interpreters group are described in [here](../manual/interpreters.html).
### List of Registered Interpreters
### List of registered interpreters
<table class="table-configuration">
<col width="200">
@ -102,8 +105,8 @@ limitations under the License.
</table>
<br/>
### List of registered interpreter settings
### List of Interpreter Settings
<table class="table-configuration">
<col width="200">
<tr>
@ -177,7 +180,7 @@ limitations under the License.
</table>
<br/>
### Create an Interpreter Setting
### Create a new interpreter setting
<table class="table-configuration">
<col width="200">
@ -256,10 +259,8 @@ limitations under the License.
</tr>
</table>
<br/>
### Update an Interpreter Setting
### Update an interpreter setting
<table class="table-configuration">
<col width="200">
<tr>
@ -337,9 +338,8 @@ limitations under the License.
</tr>
</table>
<br/>
### Delete an Interpreter Setting
### Delete an interpreter setting
<table class="table-configuration">
<col width="200">
@ -369,7 +369,7 @@ limitations under the License.
<br/>
### Restart an Interpreter
### Restart an interpreter
<table class="table-configuration">
<col width="200">
@ -398,7 +398,7 @@ limitations under the License.
</table>
<br/>
### Add repository for dependency resolving
### Add a new repository for dependency resolving
<table class="table-configuration">
<col width="200">
@ -439,7 +439,7 @@ limitations under the License.
</table>
<br/>
### Delete repository for dependency resolving
### Delete a repository for dependency resolving
<table class="table-configuration">
<col width="200">
@ -460,3 +460,4 @@ limitations under the License.
<td> 500 </td>
</tr>
</table>

71
docs/rest-api/rest-notebook.md
View file

@ -19,20 +19,23 @@ limitations under the License.
-->
{% include JB/setup %}
## Zeppelin REST API
Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
# Apache Zeppelin Notebook REST API
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`. Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
<div id="toc"></div>
If you work with Zeppelin and find a need for an additional REST API, please [file an issue or send us mail](../../community.html).
## Overview
Apache Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html).
<br />
## Notebook REST API List
Notebooks REST API supports the following operations: List, Create, Get, Delete, Clone, Run, Export, Import as detailed in the following tables.
### List Notebooks
### List of the notebooks
<table class="table-configuration">
<col width="200">
<tr>
@ -74,8 +77,7 @@ limitations under the License.
</table>
<br/>
### Create Notebook
### Create a new notebook
<table class="table-configuration">
<col width="200">
<tr>
@ -129,8 +131,7 @@ limitations under the License.
</table>
<br/>
### Get Notebook
### Get an existing notebook information
<table class="table-configuration">
<col width="200">
<tr>
@ -227,8 +228,7 @@ limitations under the License.
</table>
<br/>
### Delete Notebook
### Delete a notebook
<table class="table-configuration">
<col width="200">
<tr>
@ -255,8 +255,7 @@ limitations under the License.
</table>
<br/>
### Clone Notebook
### Clone a notebook
<table class="table-configuration">
<col width="200">
<tr>
@ -294,13 +293,12 @@ limitations under the License.
</table>
<br/>
### Run Notebook Job
### Run all paragraphs
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```POST``` method runs all paragraph in the given notebook id.
<td>This ```POST``` method runs all paragraphs in the given notebook id.
</td>
</tr>
<tr>
@ -322,13 +320,12 @@ limitations under the License.
</table>
<br/>
### Stop Notebook Job
### Stop all paragraphs
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```DELETE``` method stops all paragraph in the given notebook id.
<td>This ```DELETE``` method stops all paragraphs in the given notebook id.
</td>
</tr>
<tr>
@ -350,13 +347,12 @@ limitations under the License.
</table>
<br/>
### Get Notebook Job
### Get the status of all paragraphs
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```GET``` method gets all paragraph status by the given notebook id.
<td>This ```GET``` method gets the status of all paragraphs by the given notebook id.
The body field of the returned JSON contains of the array that compose of the paragraph id, paragraph status, paragraph finish date, paragraph started date.
</td>
</tr>
@ -397,8 +393,7 @@ limitations under the License.
</table>
<br/>
### Run Paragraph Job
### Run a paragraph
<table class="table-configuration">
<col width="200">
<tr>
@ -436,8 +431,7 @@ limitations under the License.
</table>
<br/>
### Stop Paragraph Job
### Stop a paragraph
<table class="table-configuration">
<col width="200">
<tr>
@ -464,7 +458,6 @@ limitations under the License.
</table>
<br/>
### Add Cron Job
<table class="table-configuration">
<col width="200">
@ -553,8 +546,7 @@ limitations under the License.
</table>
<br />
### Full Text Search Through the Paragraphs in All Notebooks
### Full text search through the paragraphs in all notebooks
<table class="table-configuration">
<col width="200">
<tr>
@ -592,8 +584,7 @@ limitations under the License.
</table>
<br/>
### Create Paragraph
### Create a new paragraph
<table class="table-configuration">
<col width="200">
<tr>
@ -643,8 +634,7 @@ limitations under the License.
</table>
<br/>
### Get Paragraph
### Get a paragraph information
<table class="table-configuration">
<col width="200">
<tr>
@ -712,8 +702,7 @@ limitations under the License.
</table>
<br/>
### Move Paragraph
### Move a paragraph to the specific index
<table class="table-configuration">
<col width="200">
<tr>
@ -741,8 +730,7 @@ limitations under the License.
<br/>
### Delete Paragraph
### Delete a paragraph
<table class="table-configuration">
<col width="200">
<tr>
@ -769,8 +757,7 @@ limitations under the License.
</table>
<br />
### Export Notebook
### Export a notebook
<table class="table-configuration">
<col width="200">
<tr>
@ -820,8 +807,7 @@ limitations under the License.
</table>
<br />
### Import Notebook
### Import a notebook
<table class="table-configuration">
<col width="200">
<tr>
@ -879,3 +865,4 @@ limitations under the License.
</tr>
</tr>
</table>

15
docs/security/authentication.md
View file

@ -19,11 +19,12 @@ limitations under the License.
-->
# Authentication for NGINX
<div id="toc"></div>
Authentication is company-specific.
One option is to use [Basic Access Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
One option is to use [Basic Access Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication)
### HTTP Basic Authentication using NGINX
## HTTP Basic Authentication using NGINX
> **Quote from Wikipedia:** NGINX is a web server. It can act as a reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer and an HTTP cache.
@ -39,7 +40,7 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
```
$ apt-get install nginx
```
*Important: On pre 1.3.13 version of NGINX, Proxy for Websocket may not fully works. Please use latest version of NGINX. See: [NGINX documentation](https://www.nginx.com/blog/websocket-nginx/)*
> **NOTE :** On pre 1.3.13 version of NGINX, Proxy for Websocket may not fully works. Please use latest version of NGINX. See: [NGINX documentation](https://www.nginx.com/blog/websocket-nginx/).
1. Setup init script in NGINX
@ -119,12 +120,12 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
1. More security consideration
* Using HTTPS connection with Basic Authentication is highly recommended since basic auth without encryption may expose your important credential information over the network.
* Using [Shiro Security feature built-into Zeppelin](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md) is recommended if you prefer all-in-one solution for authentication but NGINX may provides ad-hoc solution for re-use authentication served by your system's NGINX server or in case of you need to separate authentication from zeppelin server.
* Using [Shiro Security feature built-into Zeppelin](./shiroauthentication.html) is recommended if you prefer all-in-one solution for authentication but NGINX may provides ad-hoc solution for re-use authentication served by your system's NGINX server or in case of you need to separate authentication from zeppelin server.
* It is recommended to isolate direct connection to Zeppelin server from public internet or external services to secure your zeppelin instance from unexpected attack or problems caused by public zone.
### Another option
## Another option
Another option is to have an authentication server that can verify user credentials in an LDAP server.
If an incoming request to the Zeppelin server does not have a cookie with user information encrypted with the authentication server public key, the user
is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific URL in the Zeppelin server which sets the authentication cookie in the browser.
The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.
The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.

2
docs/security/interpreter_authorization.md
View file

@ -19,6 +19,8 @@ limitations under the License.
-->
# Interpreter and Data Source Authorization
<div id="toc"></div>
## Interpreter Authorization
Interpreter authorization involves permissions like creating an interpreter and execution queries using it.

7
docs/security/notebook_authorization.md
View file

@ -19,6 +19,9 @@ limitations under the License.
-->
# Zeppelin Notebook Authorization
<div id="toc"></div>
## Overview
We assume that there is an **Shiro Authentication** component that associates a user string and a set of group strings with every NotebookSocket.
If you don't set the authentication components yet, please check [Shiro authentication for Apache Zeppelin](./shiroauthentication.html) first.
@ -44,12 +47,12 @@ If someone who doesn't have **read** permission is trying to access the notebook
## How it works
In this section, we will explain the detail about how the notebook authorization works in backend side.
#### NotebookServer
### NotebookServer
The [NotebookServer](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java) classifies every notebook operations into three categories: **Read**, **Write**, **Manage**.
Before executing a notebook operation, it checks if the user and the groups associated with the `NotebookSocket` have permissions.
For example, before executing a **Read** operation, it checks if the user and the groups have at least one entity that belongs to the **Reader** entities.
#### Notebook REST API call
### Notebook REST API call
Zeppelin executes a [REST API call](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java) for the notebook permission information.
In the backend side, Zeppelin gets the user information for the connection and allows the operation if the users and groups
associated with the current user have at least one entity that belongs to owner entities for the notebook.

52
docs/security/shiroauthentication.md
View file

@ -20,6 +20,10 @@ limitations under the License.
{% include JB/setup %}
# Shiro authentication for Apache Zeppelin
<div id="toc"></div>
## Overview
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
When you connect to Apache Zeppelin, you will be asked to enter your credentials. Once you logged in, then you have access to all notes including other user's notes.
@ -27,28 +31,28 @@ When you connect to Apache Zeppelin, you will be asked to enter your credentials
## Security Setup
You can setup **Zeppelin notebook authentication** in some simple steps.
####1. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
### 1. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authc** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authc** means "formed auth security".
The default status of them is
```
/** = anon
#/** = authcBasic
#/** = authc
```
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
Deactivate the line "/** = anon" and activate the line "/** = authc" in `conf/shiro.ini` file.
```
#/** = anon
/** = authcBasic
/** = authc
```
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
####2. Secure the Websocket channel
### 2. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
####3. Start Zeppelin
### 3. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
@ -56,7 +60,7 @@ bin/zeppelin-daemon.sh start (or restart)
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
### 4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
@ -67,6 +71,36 @@ user1 = password2
user2 = password3
```
Those combinations are defined in the `conf/shiro.ini` file.
### 5. Groups and permissions (optional)
In case you want to leverage user groups and permissions, use one of the following configuration for LDAP or AD under `[main]` segment in `shiro.ini`
```
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
activeDirectoryRealm.systemUsername = userNameA
activeDirectoryRealm.systemPassword = passwordA
activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
activeDirectoryRealm.url = ldap://ldap.test.com:389
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
activeDirectoryRealm.authorizationCachingEnabled = false
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
# search base for ldap groups (only relevant for LdapGroupRealm):
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
ldapRealm.contextFactory.url = ldap://ldap.test.com:389
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
```
also define roles/groups that you want to have in system, like below;
```
[roles]
admin = *
hr = *
finance = *
group1 = *
```
All of above configurations are defined in the `conf/shiro.ini` file.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).

23
docs/storage/storage.md
View file

@ -17,14 +17,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
## Notebook Storage
# Notebook storage options for Apache Zeppelin
Zeppelin has a pluggable notebook storage mechanism controlled by `zeppelin.notebook.storage` configuration option with multiple implementations.
There are few Notebook storage systems available for a use out of the box:
- (default) all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
- use local file system and version it using local Git repository - `GitNotebookRepo`
- storage using Amazon S3 service - `S3NotebookRepo`
- storage using Azure service - `AzureNotebookRepo`
<div id="toc"></div>
## Overview
Apache Zeppelin has a pluggable notebook storage mechanism controlled by `zeppelin.notebook.storage` configuration option with multiple implementations.
There are few notebook storage systems available for a use out of the box:
* (default) all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
* use local file system and version it using local Git repository - `GitNotebookRepo`
* storage using Amazon S3 service - `S3NotebookRepo`
* storage using Azure service - `AzureNotebookRepo`
Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration.
By default, only first two of them will be automatically kept in sync by Zeppelin.
@ -146,8 +151,6 @@ Or using the following setting in **zeppelin-site.xml**:
Using `AzureNotebookRepo` you can connect your Zeppelin with your Azure account for notebook storage.
</br>
First of all, input your `AccountName`, `AccountKey`, and `Share Name` in the file **zeppelin-site.xml** by commenting out and completing the next properties:
```
@ -233,4 +236,4 @@ export ZEPPELINHUB_API_TOKEN = ZeppelinHub token
export ZEPPELINHUB_API_ADDRESS = address of ZeppelinHub service (e.g. https://www.zeppelinhub.com)
```
You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).

11
elasticsearch/pom.xml
View file

@ -34,7 +34,7 @@
<url>http://www.apache.org</url>
<properties>
<elasticsearch.version>2.1.0</elasticsearch.version>
<elasticsearch.version>2.3.3</elasticsearch.version>
<guava.version>18.0</guava.version>
<json-flattener.version>0.1.6</json-flattener.version>
</properties>
@ -79,15 +79,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

5
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java
View file

@ -37,6 +37,7 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
@ -244,8 +245,8 @@ public class ElasticsearchInterpreter extends Interpreter {
}
@Override
public List<String> completion(String s, int i) {
final List<String> suggestions = new ArrayList<>();
public List<InterpreterCompletion> completion(String s, int i) {
final List suggestions = new ArrayList<>();
if (StringUtils.isEmpty(s)) {
suggestions.addAll(COMMANDS);

9
file/pom.xml
View file

@ -72,15 +72,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>

3
file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java
View file

@ -23,6 +23,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -165,7 +166,7 @@ public abstract class FileInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}
}

5
file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java
View file

@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
/**
* HDFS implementation of File interpreter for Zeppelin.
@ -259,9 +260,9 @@ public class HDFSFileInterpreter extends FileInterpreter {
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
logger.info("Completion request at position\t" + cursor + " in string " + buf);
final List<String> suggestions = new ArrayList<>();
final List suggestions = new ArrayList<>();
if (StringUtils.isEmpty(buf)) {
suggestions.add("ls");
suggestions.add("cd");

11
flink/pom.xml
View file

@ -35,7 +35,7 @@
<url>http://zeppelin.apache.org</url>
<properties>
<flink.version>1.0.0</flink.version>
<flink.version>1.0.3</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<flink.scala.binary.version>2.10</flink.scala.binary.version>
<flink.scala.version>2.10.4</flink.scala.version>
@ -294,15 +294,6 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

5
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -38,6 +38,7 @@ import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -331,8 +332,8 @@ public class FlinkInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
return new LinkedList<String>();
public List<InterpreterCompletion> completion(String buf, int cursor) {
return new LinkedList<>();
}
private void startFlinkMiniCluster() {

9
geode/pom.xml
View file

@ -84,15 +84,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

3
geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java
View file

@ -24,6 +24,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -300,7 +301,7 @@ public class GeodeOqlInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}

9
hbase/pom.xml
View file

@ -109,15 +109,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

3
hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
View file

@ -15,6 +15,7 @@
package org.apache.zeppelin.hbase;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.jruby.embed.LocalContextScope;
@ -152,7 +153,7 @@ public class HbaseInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}

9
ignite/pom.xml
View file

@ -107,15 +107,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

3
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -23,6 +23,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -342,7 +343,7 @@ public class IgniteInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return new LinkedList<>();
}

3
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java
View file

@ -22,6 +22,7 @@ import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -194,7 +195,7 @@ public class IgniteSqlInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return new LinkedList<>();
}
}

9
jdbc/pom.xml
View file

@ -102,15 +102,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

39
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -31,10 +31,12 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -98,6 +100,9 @@ public class JDBCInterpreter extends Interpreter {
static final String EMPTY_COLUMN_VALUE = "";
private final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use";
private final String CONCURRENT_EXECUTION_COUNT = "zeppelin.jdbc.concurrent.max_connection";
private final HashMap<String, Properties> propertiesMap;
private final Map<String, Statement> paragraphIdStatementMap;
@ -113,7 +118,7 @@ public class JDBCInterpreter extends Interpreter {
}
};
private static final List<String> NO_COMPLETION = new ArrayList<>();
private static final List<InterpreterCompletion> NO_COMPLETION = new ArrayList<>();
public JDBCInterpreter(Properties property) {
super(property);
@ -357,12 +362,11 @@ public class JDBCInterpreter extends Interpreter {
return new InterpreterResult(Code.SUCCESS, msg.toString());
} catch (SQLException ex) {
logger.error("Cannot run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
} catch (ClassNotFoundException e) {
} catch (Exception e) {
logger.error("Cannot run " + sql, e);
return new InterpreterResult(Code.ERROR, e.getMessage());
StringBuilder stringBuilder = new StringBuilder(e.getClass().toString()).append("\n");
stringBuilder.append(StringUtils.join(e.getStackTrace(), "\n"));
return new InterpreterResult(Code.ERROR, stringBuilder.toString());
}
}
@ -433,16 +437,19 @@ public class JDBCInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
JDBCInterpreter.class.getName() + this.hashCode());
String schedulerName = JDBCInterpreter.class.getName() + this.hashCode();
return isConcurrentExecution() ?
SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName, 10)
: SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName);
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
List<CharSequence> candidates = new ArrayList<>();
SqlCompleter sqlCompleter = propertyKeySqlCompleterMap.get(getPropertyKey(buf));
if (sqlCompleter != null && sqlCompleter.complete(buf, cursor, candidates) >= 0) {
return Lists.transform(candidates, sequenceToStringTransformer);
List completion = Lists.transform(candidates, sequenceToStringTransformer);
return completion;
} else {
return NO_COMPLETION;
}
@ -452,5 +459,17 @@ public class JDBCInterpreter extends Interpreter {
return Integer.valueOf(
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
}
boolean isConcurrentExecution() {
return Boolean.valueOf(getProperty(CONCURRENT_EXECUTION_KEY));
}
int getMaxConcurrentConnection() {
try {
return Integer.valueOf(getProperty(CONCURRENT_EXECUTION_COUNT));
} catch (Exception e) {
return 10;
}
}
}

12
jdbc/src/main/resources/interpreter-setting.json
View file

@ -33,6 +33,18 @@
"propertyName": "common.max_count",
"defaultValue": "1000",
"description": "Max number of SQL result to display."
},
"zeppelin.jdbc.concurrent.use": {
"envName": null,
"propertyName": "zeppelin.jdbc.concurrent.use",
"defaultValue": "true",
"description": "Use parallel scheduler"
},
"zeppelin.jdbc.concurrent.max_connection": {
"envName": null,
"propertyName": "zeppelin.jdbc.concurrent.max_connection",
"defaultValue": "10",
"description": "Number of concurrent execution"
}
}
}

28
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -22,6 +22,8 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
@ -32,6 +34,9 @@ import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.jdbc.JDBCInterpreter;
import org.apache.zeppelin.scheduler.FIFOScheduler;
import org.apache.zeppelin.scheduler.ParallelScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.junit.Before;
import org.junit.Test;
@ -200,4 +205,27 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message());
}
@Test
public void concurrentSettingTest() {
Properties properties = new Properties();
properties.setProperty("zeppelin.jdbc.concurrent.use", "true");
properties.setProperty("zeppelin.jdbc.concurrent.max_connection", "10");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
assertTrue(jdbcInterpreter.isConcurrentExecution());
assertEquals(10, jdbcInterpreter.getMaxConcurrentConnection());
Scheduler scheduler = jdbcInterpreter.getScheduler();
assertTrue(scheduler instanceof ParallelScheduler);
properties.clear();
properties.setProperty("zeppelin.jdbc.concurrent.use", "false");
jdbcInterpreter = new JDBCInterpreter(properties);
assertFalse(jdbcInterpreter.isConcurrentExecution());
scheduler = jdbcInterpreter.getScheduler();
assertTrue(scheduler instanceof FIFOScheduler);
}
}

9
kylin/pom.xml
View file

@ -61,15 +61,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

3
kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java
View file

@ -28,6 +28,7 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -112,7 +113,7 @@ public class KylinInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}

Some files were not shown because too many files have changed in this diff Show more