Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/master' into livyInterperter

Browse source
This commit is contained in:
Prabhjyot Singh 2016-03-27 18:30:59 +05:30
commit 6c6b164046
46 changed files with 853 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -18,6 +18,7 @@ lens/lens-cli-hist.log
# conf file
conf/zeppelin-env.sh
conf/zeppelin-env.cmd
conf/zeppelin-site.xml
conf/keystore
conf/truststore

112
bin/common.cmd Normal file
View file

@ -0,0 +1,112 @@
@echo off
REM Licensed to the Apache Software Foundation (ASF) under one or more
REM contributor license agreements. See the NOTICE file distributed with
REM this work for additional information regarding copyright ownership.
REM The ASF licenses this file to You under the Apache License, Version 2.0
REM (the "License"); you may not use this file except in compliance with
REM the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing, software
REM distributed under the License is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM See the License for the specific language governing permissions and
REM limitations under the License.
if not defined ZEPPELIN_HOME (
for %%d in ("%~dp0..") do (
set ZEPPELIN_HOME=%%~fd
)
)
if not defined ZEPPELIN_CONF_DIR (
set ZEPPELIN_CONF_DIR=%ZEPPELIN_HOME%\conf
)
if not defined ZEPPELIN_LOG_DIR (
set ZEPPELIN_LOG_DIR=%ZEPPELIN_HOME%\logs
)
if not defined ZEPPELIN_NOTEBOOK_DIR (
set ZEPPELIN_NOTEBOOK_DIR=%ZEPPELIN_HOME%\notebook
)
if not defined ZEPPELIN_PID_DIR (
set ZEPPELIN_PID_DIR=%ZEPPELIN_HOME%\run
)
if not defined ZEPPELIN_WAR (
if exist "%ZEPPELIN_HOME%\zeppelin-web\dist" (
set ZEPPELIN_WAR=%ZEPPELIN_HOME%\zeppelin-web\dist
) else (
for %%d in ("%ZEPPELIN_HOME%\zeppelin-web*.war") do (
set ZEPPELIN_WAR=%%d
)
)
)
if not defined ZEPPELIN_INTERPRETER_DIR (
set ZEPPELIN_INTERPRETER_DIR=%ZEPPELIN_HOME%\interpreter
)
if exist "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" (
call "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd"
)
if not defined ZEPPELIN_CLASSPATH (
set ZEPPELIN_CLASSPATH="%ZEPPELIN_CONF_DIR%"
) else (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_CONF_DIR%"
)
if not defined ZEPPELIN_ENCODING (
set ZEPPELIN_ENCODING=UTF-8
)
if not defined ZEPPELIN_MEM (
set ZEPPELIN_MEM=-Xms1024m -Xmx1024m -XX:MaxPermSize=512m
)
if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=-Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
) else (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)
if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_JAVA_OPTS (
set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
)
if not defined ZEPPELIN_INTP_MEM (
set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM%
)
set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING%
if not defined JAVA_HOME (
set ZEPPELIN_RUNNER=java
) else (
set ZEPPELIN_RUNNER=%JAVA_HOME%\bin\java
)
if not defined ZEPPELIN_IDENT_STRING (
set ZEPPELIN_IDENT_STRING=%USERNAME%
)
if not defined DEBUG (
set DEBUG=0
)
if not defined ZEPPELIN_INTERPRETER_REMOTE_RUNNER (
set ZEPPELIN_INTERPRETER_REMOTE_RUNNER=bin\interpreter.cmd
)
exit /b

12
bin/common.sh
View file

@ -81,6 +81,18 @@ function addJarInDir(){
fi
}
ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils
function getZeppelinVersion(){
if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
$ZEPPELIN_RUNNER -cp $CLASSPATH $ZEPPELIN_COMMANDLINE_MAIN -v
exit 0
}
# Text encoding for
# read/write job into files,
# receiving/displaying query/result.

38
bin/functions.cmd Normal file
View file

@ -0,0 +1,38 @@
@echo off
REM Licensed to the Apache Software Foundation (ASF) under one or more
REM contributor license agreements. See the NOTICE file distributed with
REM this work for additional information regarding copyright ownership.
REM The ASF licenses this file to You under the Apache License, Version 2.0
REM (the "License"); you may not use this file except in compliance with
REM the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing, software
REM distributed under the License is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM See the License for the specific language governing permissions and
REM limitations under the License.
if not "%1"=="" goto %1
exit /b
:ADDEACHJARINDIR
for %%d in ("%~2\*.jar") do (
set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH!
)
exit /b
:ADDEACHJARINDIRRECURSIVE
for /r "%~2" %%d in (*.jar) do (
set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH!
)
exit /b
:ADDJARINDIR
if exist "%~2" (
set ZEPPELIN_CLASSPATH="%~2\*";%ZEPPELIN_CLASSPATH%
)
exit /b

136
bin/interpreter.cmd Normal file
View file

@ -0,0 +1,136 @@
@echo off
REM Licensed to the Apache Software Foundation (ASF) under one or more
REM contributor license agreements. See the NOTICE file distributed with
REM this work for additional information regarding copyright ownership.
REM The ASF licenses this file to You under the Apache License, Version 2.0
REM (the "License"); you may not use this file except in compliance with
REM the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing, software
REM distributed under the License is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM See the License for the specific language governing permissions and
REM limitations under the License.
setlocal enableextensions enabledelayedexpansion
set bin=%~dp0
:loop
if "%~1"=="" goto cont
if /I "%~1"=="-h" goto usage
if /I "%~1"=="-d" (
set INTERPRETER_DIR=%~2
set INTERPRETER_ID=%~n2
)
if /I "%~1"=="-p" set PORT=%~2
if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2
shift
goto loop
:cont
if "%PORT%"=="" goto usage
if "%INTERPRETER_DIR%"=="" goto usage
call "%bin%\common.cmd"
if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes"
) else (
for %%d in ("%ZEPPELIN_HOME%\lib\zeppelin-interpreter*.jar") do (
set ZEPPELIN_INTERPRETER_JAR=%%d
)
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"!ZEPPELIN_INTERPRETER_JAR!"
)
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib"
call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%"
set HOSTNAME=%COMPUTERNAME%
set ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-interpreter-%INTERPRETER_ID%-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log
if not exist "%ZEPPELIN_LOG_DIR%" (
echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR%
mkdir "%ZEPPELIN_LOG_DIR%"
)
if /I "%INTERPRETER_ID%"=="spark" (
if defined SPARK_HOME (
set SPARK_SUBMIT=%SPARK_HOME%\bin\spark-submit.cmd
for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\zeppelin-spark*.jar") do (
set SPARK_APP_JAR=%%d
)
set ZEPPELIN_CLASSPATH="!SPARK_APP_JAR!"
for %%d in ("%SPARK_HOME%\python\lib\py4j-*-src.zip") do (
set py4j=%%d
)
if not defined PYTHONPATH (
set PYTHONPATH=!py4j!;%SPARK_HOME%\python
) else (
set PYTHONPATH=!py4j!;%SPARK_HOME%\python;%PYTHONPATH%
)
) else (
if defined HADOOP_HOME if exist "%HADOOP_HOME%\bin\hadoop.cmd" (
for /f "tokens=*" %%d in ('"%HADOOP_HOME%\bin\hadoop.cmd" classpath') do (
set LOCAL_HADOOP_CLASSPATH=%%d
)
set ZEPPELIN_CLASSPATH=!LOCAL_HADOOP_CLASSPATH!;%ZEPPELIN_CLASSPATH%
)
call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\dep"
for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\pyspark\py4j-*-src.zip") do (
set py4j=%%d
)
set PYSPARKPATH=%ZEPPELIN_HOME%\interpreter\spark\pyspark\pyspark.zip;!py4j!
if not defined PYTHONPATH (
set PYTHONPATH=!PYSPARKPATH!
) else (
set PYTHONPATH=%PYTHONPATH%;!PYSPARKPATH!
)
set PYSPARKPATH=
if defined HADOOP_HOME if not defined HADOOP_CONF_DIR (
if exist "%HADOOP_HOME%\etc\hadoop" (
set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
)
)
if exist "%HADOOP_CONF_DIR%" (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%HADOOP_CONF_DIR%"
)
)
)
call "%bin%\functions.cmd" ADDJARINDIR "%LOCAL_INTERPRETER_REPO%"
if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
set CLASSPATH=%ZEPPELIN_CLASSPATH%
) else (
set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH%
)
if defined SPARK_SUBMIT (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%'
"%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" %PORT%
) else (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"
"%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% %PORT%
)
exit /b
:usage
echo Usage: %~n0 -p ^<port^> -d ^<interpreter dir to load^> -l ^<local interpreter repo dir to load^>

6
bin/interpreter.sh
View file

@ -23,7 +23,7 @@ function usage() {
echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
}
while getopts "hp:d:l:" o; do
while getopts "hp:d:l:v" o; do
case ${o} in
h)
usage
@ -38,6 +38,10 @@ while getopts "hp:d:l:" o; do
l)
LOCAL_INTERPRETER_REPO=${OPTARG}
;;
v)
. "${bin}/common.sh"
getZeppelinVersion
;;
esac
done

7
bin/zeppelin-daemon.sh
View file

@ -19,7 +19,9 @@
# description: Start and stop daemon script for.
#
USAGE="Usage: zeppelin-daemon.sh [--config <conf-dir>] {start|stop|upstart|restart|reload|status}"
USAGE="-e Usage: zeppelin-daemon.sh\n\t
[--config <conf-dir>] {start|stop|upstart|restart|reload|status}\n\t
[--version | -v]"
if [[ "$1" == "--config" ]]; then
shift
@ -258,6 +260,9 @@ case "${1}" in
status)
find_zeppelin_process
;;
-v | --version)
getZeppelinVersion
;;
*)
echo ${USAGE}
esac

91
bin/zeppelin.cmd Normal file
View file

@ -0,0 +1,91 @@
@echo off
REM Licensed to the Apache Software Foundation (ASF) under one or more
REM contributor license agreements. See the NOTICE file distributed with
REM this work for additional information regarding copyright ownership.
REM The ASF licenses this file to You under the Apache License, Version 2.0
REM (the "License"); you may not use this file except in compliance with
REM the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing, software
REM distributed under the License is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM See the License for the specific language governing permissions and
REM limitations under the License.
setlocal enableextensions enabledelayedexpansion
set bin=%~dp0
if not "%1"=="--config" goto MAIN
:SET_CONFIG
shift
set conf_dir=%~f1
shift
if not exist "%conf_dir%" (
echo ERROR: %conf_dir% is not a directory
echo Usage: %~n0 [--config ^<conf-dir^>]
exit /b 1
) else (
set ZEPPELIN_CONF_DIR=%conf_dir%
)
:MAIN
call "%bin%\common.cmd"
set HOSTNAME=%COMPUTERNAME%
set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log
set ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer
set JAVA_OPTS=%JAVA_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"
if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes"
)
if exist "%ZEPPELIN_HOME%\zeppelin-zengine\target\classes" (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-zengine\target\classes"
)
if exist "%ZEPPELIN_HOME%\zeppelin-server\target\classes" (
set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-server\target\classes"
)
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%"
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\lib"
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib"
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-zengine\target\lib"
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-server\target\lib"
call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-web\target\lib"
if not defined CLASSPATH (
set CLASSPATH=%ZEPPELIN_CLASSPATH%
) else (
set CLASSPATH=%CLASSPATH%;%ZEPPELIN_CLASSPATH%
)
if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
set CLASSPATH=%ZEPPELIN_CLASSPATH%
) else (
set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH%
)
if not exist %ZEPPELIN_LOG_DIR% (
echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR%
mkdir "%ZEPPELIN_LOG_DIR%"
)
if not exist %ZEPPELIN_PID_DIR% (
echo Pid dir doesn't exist, create %ZEPPELIN_PID_DIR%
mkdir "%ZEPPELIN_PID_DIR%"
)
if not exist %ZEPPELIN_NOTEBOOK_DIR% (
echo Notebook dir doesn't exist, create %ZEPPELIN_NOTEBOOK_DIR%
mkdir "%ZEPPELIN_NOTEBOOK_DIR%"
)
"%ZEPPELIN_RUNNER%" %JAVA_OPTS% -cp %CLASSPATH% %ZEPPELIN_SERVER% "%*"

4
bin/zeppelin.sh
View file

@ -39,6 +39,10 @@ bin=$(cd "${bin}">/dev/null; pwd)
. "${bin}/common.sh"
if [ "$1" == "--version" ] || [ "$1" == "-v" ]; then
getZeppelinVersion
fi
HOSTNAME=$(hostname)
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log"
LOG="${ZEPPELIN_LOG_DIR}/zeppelin-cli-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out"

64
conf/zeppelin-env.cmd.template Normal file
View file

@ -0,0 +1,64 @@
@echo off
REM Licensed to the Apache Software Foundation (ASF) under one or more
REM contributor license agreements. See the NOTICE file distributed with
REM this work for additional information regarding copyright ownership.
REM The ASF licenses this file to You under the Apache License, Version 2.0
REM (the "License"); you may not use this file except in compliance with
REM the License. You may obtain a copy of the License at
REM
REM http://www.apache.org/licenses/LICENSE-2.0
REM
REM Unless required by applicable law or agreed to in writing, software
REM distributed under the License is distributed on an "AS IS" BASIS,
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
REM See the License for the specific language governing permissions and
REM limitations under the License.
REM
REM set JAVA_HOME=
REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default.
REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default.
REM set ZEPPELIN_WAR_TEMPDIR REM The location of jetty temporary directory.
REM set ZEPPELIN_NOTEBOOK_DIR REM Where notebook saved
REM set ZEPPELIN_NOTEBOOK_HOMESCREEN REM Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
REM set ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE REM hide homescreen notebook from list when this value set to "true". default "false"
REM set ZEPPELIN_NOTEBOOK_S3_BUCKET REM Bucket where notebook saved
REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM Spark interpreter configuration
REM Use provided spark installation
REM defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit
REM
REM set SPARK_HOME REM (required) When it is defined, load it instead of Zeppelin embedded Spark libraries
REM set SPARK_SUBMIT_OPTIONS REM (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G".
REM set SPARK_APP_NAME REM (optional) The name of spark application.
REM Use embedded spark binaries
REM without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries.
REM however, it is not encouraged when you can define SPARK_HOME
REM
REM Options read in YARN client mode
REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR.
REM Pyspark (supported with Spark 1.2.1 and above)
REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers.
REM set PYTHONPATH
REM Spark interpreter options
REM
REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default.
REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default.
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default.

5
conf/zeppelin-env.sh.template
View file

@ -29,8 +29,9 @@
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false"
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading

6
conf/zeppelin-site.xml.template
View file

@ -76,6 +76,12 @@
<description>bucket name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.s3.endpoint</name>
<value>s3.amazonaws.com</value>
<description>endpoint for s3 bucket</description>
</property>
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.S3NotebookRepo</value>

2
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -41,6 +41,7 @@
<ul class="dropdown-menu">
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/interpreter/alluxio.html">Alluxio</a></li>
<li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
<li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
@ -56,7 +57,6 @@
<li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Shell</a></li>
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
<li><a href="{{BASE_PATH}}/interpreter/alluxio.html">Alluxio</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Tajo</a></li>
<li role="separator" class="divider"></li>
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>

19
docs/install/install.md
View file

@ -45,7 +45,7 @@ If you don't have requirements prepared, please check instructions in [README.md
<a name="zeppelin-configuration"> </a>
## Zeppelin Configuration
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` and **java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will be used priorly.
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
<table class="table-configuration">
<tr>
@ -186,6 +186,12 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>user</td>
<td>A user name of S3 bucket<br />i.e. <code>bucket/user/notebook/2A94M5J1Z/note.json</code></td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_ENDPOINT</td>
<td>zeppelin.notebook.s3.endpoint</td>
<td>s3.amazonaws.com</td>
<td>Endpoint for the bucket</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>
@ -228,7 +234,7 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
</table>
Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation.
[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example.
[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example.
## Zeppelin Start / Stop
#### Start Zeppelin
@ -248,9 +254,9 @@ bin/zeppelin-daemon.sh stop
Zeppelin can auto start as a service with an init script, such as services managed by upstart.
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
This example has been tested with Ubuntu Linux.
This also allows the service to be managed with commands such as
This also allows the service to be managed with commands such as
`sudo service zeppelin start`
`sudo service zeppelin stop`
@ -278,4 +284,9 @@ chdir /usr/share/zeppelin
exec bin/zeppelin-daemon.sh upstart
```
#### Running on Windows
```
bin\zeppelin.cmd
```

2
docs/interpreter/spark.md
View file

@ -126,6 +126,8 @@ export HADOOP_CONF_DIR=/usr/lib/hadoop
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"
```
For Windows, ensure you have `winutils.exe` in `%HADOOP_HOME%\bin`. For more details please see [Problems running Hadoop on Windows](https://wiki.apache.org/hadoop/WindowsProblems)
### 2. Set master in Interpreter menu
After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type.

21
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -19,17 +19,14 @@ package org.apache.zeppelin.shell;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
@ -50,6 +47,10 @@ public class ShellInterpreter extends Interpreter {
public static final String SHELL_COMMAND_TIMEOUT = "shell.command.timeout.millisecs";
public static final String DEFAULT_COMMAND_TIMEOUT = "600000";
int commandTimeOut;
private static final boolean isWindows = System
.getProperty("os.name")
.startsWith("Windows");
final String shell = isWindows ? "cmd /c" : "bash -c";
static {
Interpreter.register(
@ -83,11 +84,15 @@ public class ShellInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.debug("Run shell command '" + cmd + "'");
CommandLine cmdLine = CommandLine.parse("bash");
cmdLine.addArgument("-c", false);
CommandLine cmdLine = CommandLine.parse(shell);
// the Windows CMD shell doesn't handle multiline statements,
// they need to be delimited by '&&' instead
if (isWindows) {
String[] lines = StringUtils.split(cmd, "\n");
cmd = StringUtils.join(lines, " && ");
}
cmdLine.addArgument(cmd, false);
DefaultExecutor executor = new DefaultExecutor();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeOut));

6
zeppelin-distribution/src/assemble/distribution.xml
View file

@ -69,6 +69,12 @@
</fileSet>
<fileSet>
<directory>../conf</directory>
<excludes>
<exclude>interpreter.json</exclude>
<exclude>zeppelin-env.cmd</exclude>
<exclude>zeppelin-env.sh</exclude>
<exclude>zeppelin-site.xml</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>../interpreter</directory>

2
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -1,6 +1,6 @@
(Apache 2.0) nvd3.js v1.7.1 (http://nvd3.org/) - https://github.com/novus/nvd3/blob/v1.7.1/LICENSE.md
(Apache 2.0) gson v2.2 (com.google.code.gson:gson:jar:2.2 - https://github.com/google/gson) - https://github.com/google/gson/blob/gson-2.2/LICENSE
(Apache 2.0) Amazon Web Services SDK for Java v1.10.1 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.10.1/LICENSE.txt
(Apache 2.0) Amazon Web Services SDK for Java v1.10.62 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.10.62/LICENSE.txt
(Apache 2.0) JavaEWAH v0.7.9 (https://github.com/lemire/javaewah) - https://github.com/lemire/javaewah/blob/master/LICENSE-2.0.txt

7
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -35,6 +35,11 @@ import org.junit.Before;
import org.junit.Test;
public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
private RemoteInterpreter intp;
@ -63,7 +68,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
p,
"note",
MockInterpreterAngular.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,

6
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -38,6 +38,10 @@ import static org.junit.Assert.assertEquals;
* Test for remote interpreter output stream
*/
public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProcessListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
@ -61,7 +65,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
new Properties(),
"note",
MockInterpreterOutputStream.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,

8
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
View file

@ -28,12 +28,16 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.junit.Test;
public class RemoteInterpreterProcessTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
@Test
public void testStartStop() {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess(
"../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(),
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null);
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
@ -50,7 +54,7 @@ public class RemoteInterpreterProcessTest {
public void testClientFactory() throws Exception {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess(
"../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap<String, String>(),
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class), 10 * 1000);
rip.reference(intpGroup);
assertEquals(0, rip.getNumActiveClient());

13
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

@ -53,6 +53,11 @@ import com.google.gson.reflect.TypeToken;
public class RemoteInterpreterTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
@ -78,7 +83,7 @@ public class RemoteInterpreterTest {
p,
noteId,
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
@ -95,7 +100,7 @@ public class RemoteInterpreterTest {
p,
noteId,
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
@ -193,7 +198,7 @@ public class RemoteInterpreterTest {
p,
"note",
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
@ -208,7 +213,7 @@ public class RemoteInterpreterTest {
p,
"note",
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,

8
zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -39,6 +39,10 @@ import static org.junit.Assert.assertTrue;
* Unittest for DistributedResourcePool
*/
public class DistributedResourcePoolTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup1;
private InterpreterGroup intpGroup2;
private HashMap<String, String> env;
@ -60,7 +64,7 @@ public class DistributedResourcePoolTest {
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
@ -77,7 +81,7 @@ public class DistributedResourcePoolTest {
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,

8
zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
View file

@ -46,6 +46,10 @@ import org.junit.Test;
public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private SchedulerFactory schedulerSvc;
private static final int TICK_WAIT = 100;
private static final int MAX_WAIT_CYCLES = 100;
@ -71,7 +75,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
p,
"note",
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
@ -159,7 +163,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
p,
"note",
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,

9
zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.rest;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.util.Util;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.core.Response;
@ -41,4 +44,10 @@ public class ZeppelinRestApi {
public Response getRoot() {
return Response.ok().build();
}
@GET
@Path("version")
public Response getVersion() {
return new JsonResponse<>(Response.Status.OK, "Zeppelin version", Util.getVersion()).build();
}
}

41
zeppelin-server/src/main/java/org/apache/zeppelin/utils/CommandLineUtils.java Normal file
View file

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.utils;
import org.apache.zeppelin.util.Util;
import java.util.Locale;
/**
* CommandLine Support Class
*/
public class CommandLineUtils {
public static void main(String[] args) {
if (args.length == 0) {
return;
}
String usage = args[0].toLowerCase(Locale.US);
switch (usage){
case "--version":
case "-v":
System.out.println(Util.getVersion());
break;
default:
}
}
}

2
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -418,4 +418,4 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
}
}
}

72
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
View file

@ -17,13 +17,11 @@
package org.apache.zeppelin.integration;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.*;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
@ -31,7 +29,9 @@ import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Test Zeppelin with web browser.
@ -246,4 +246,68 @@ public class ZeppelinIT extends AbstractZeppelinIT {
handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
}
}
@Test
public void testAngularRunParagraph() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
createNewNote();
// wait for first paragraph's " READY " status text
waitForParagraph(1, "READY");
// Create 1st paragraph
setTextOfParagraph(1,
"%angular <div id=\\'angularRunParagraph\\'>Run second paragraph</div>");
runParagraph(1);
waitForParagraph(1, "FINISHED");
waitForText("Run second paragraph", By.xpath(
getParagraphXPath(1) + "//div[@id=\"angularRunParagraph\"]"));
// Create 2nd paragraph
setTextOfParagraph(2, "%sh echo TEST");
runParagraph(2);
waitForParagraph(2, "FINISHED");
// Get 2nd paragraph id
final String secondParagraphId = driver.findElement(By.xpath(getParagraphXPath(2)
+ "//div[@class=\"control ng-scope\"]//ul[@class=\"dropdown-menu\"]/li[1]"))
.getAttribute("textContent");
assertTrue("Cannot find paragraph id for the 2nd paragraph", isNotBlank(secondParagraphId));
// Update first paragraph to call z.runParagraph() with 2nd paragraph id
setTextOfParagraph(1,
"%angular <div id=\\'angularRunParagraph\\' ng-click=\\'z.runParagraph(\""
+ secondParagraphId.trim()
+ "\")\\'>Run second paragraph</div>");
runParagraph(1);
waitForParagraph(1, "FINISHED");
// Set new text value for 2nd paragraph
setTextOfParagraph(2, "%sh echo NEW_VALUE");
// Click on 1 paragraph to trigger z.runParagraph() function
driver.findElement(By.xpath(
getParagraphXPath(1) + "//div[@id=\"angularRunParagraph\"]")).click();
waitForParagraph(2, "FINISHED");
// Check that 2nd paragraph has been executed
waitForText("NEW_VALUE", By.xpath(
getParagraphXPath(2) + "//div[contains(@id,\"_text\") and @class=\"text\"]"));
//delete created notebook for cleanup.
deleteTestNotebook(driver);
sleep(1000, true);
LOG.info("testAngularRunParagraph Test executed");
} catch (Exception e) {
handleException("Exception in ZeppelinIT while testAngularRunParagraph", e);
}
}
}

20
zeppelin-web/bower.json
View file

@ -2,16 +2,16 @@
"name": "zeppelin-web",
"version": "0.0.0",
"dependencies": {
"angular": "1.3.8",
"angular": "1.5.0",
"json3": "~3.3.1",
"es5-shim": "~3.1.0",
"bootstrap": "~3.2.0",
"angular-cookies": "1.3.8",
"angular-sanitize": "1.3.8",
"angular-animate": "1.3.8",
"angular-touch": "1.3.8",
"angular-route": "1.3.8",
"angular-resource": "1.3.8",
"angular-cookies": "1.5.0",
"angular-sanitize": "1.5.0",
"angular-animate": "1.5.0",
"angular-touch": "1.5.0",
"angular-route": "1.5.0",
"angular-resource": "1.5.0",
"angular-bootstrap": "~0.13.0",
"angular-websocket": "~1.0.13",
"ace-builds": "1.1.9",
@ -20,20 +20,20 @@
"nvd3": "~1.7.1",
"angular-dragdrop": "~1.0.8",
"perfect-scrollbar": "~0.5.4",
"ng-sortable": "~1.1.9",
"ng-sortable": "~1.3.3",
"angular-elastic": "~2.4.2",
"angular-elastic-input": "~2.2.0",
"angular-xeditable": "0.1.8",
"highlightjs": "~8.4.0",
"lodash": "~3.9.3",
"angular-filter": "~0.5.4",
"ngtoast": "~1.5.5",
"ngtoast": "~2.0.0",
"ng-focus-if": "~1.0.2",
"bootstrap3-dialog": "bootstrap-dialog#~1.34.7",
"floatThead": "~1.3.2"
},
"devDependencies": {
"angular-mocks": "1.3.8"
"angular-mocks": "1.5.0"
},
"appPath": "src",
"resolutions": {

4
zeppelin-web/src/app/app.js
View file

@ -17,14 +17,14 @@
'use strict';
(function() {
var zeppelinWebApp = angular.module('zeppelinWebApp', [
'ngAnimate',
'ngCookies',
'ngAnimate',
'ngRoute',
'ngSanitize',
'angular-websocket',
'ui.ace',
'ui.bootstrap',
'ui.sortable',
'as.sortable',
'ngTouch',
'ngDragDrop',
'angular.filter',

13
zeppelin-web/src/app/home/home.controller.js
View file

@ -13,7 +13,7 @@
*/
'use strict';
angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv) {
angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv, $http, baseUrlSrv) {
var vm = this;
vm.notes = notebookListDataFactory;
vm.websocketMsgSrv = websocketMsgSrv;
@ -23,9 +23,20 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
vm.staticHome = false;
$scope.isReloading = false;
var getZeppelinVersion = function() {
$http.get(baseUrlSrv.getRestApiBase() +'/version').
success(function (data, status, headers, config) {
$scope.zeppelinVersion = data.body;
}).
error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
});
};
var initHome = function() {
websocketMsgSrv.getHomeNotebook();
getZeppelinVersion();
};
initHome();

4
zeppelin-web/src/app/home/home.html
View file

@ -19,7 +19,7 @@ limitations under the License.
</div>
<div style="margin-top: -380px;">
<h1 class="box-heading" id="welcome">
Welcome to Zeppelin!
Welcome to Zeppelin! <small>({{zeppelinVersion}})</small>
</h1>
Zeppelin is web-based notebook that enables interactive data analytics.<br>
You can make beautiful data-driven, interactive, collaborative document with SQL, code and even more!<br>
@ -72,7 +72,7 @@ limitations under the License.
<div ng-show="home.notebookHome" id="{{currentParagraph.id}}_paragraphColumn_main"
ng-repeat="currentParagraph in home.note.paragraphs"
ng-controller="ParagraphCtrl"
ng-Init="init(currentParagraph)"
ng-Init="init(currentParagraph, home.note)"
ng-class="columnWidthClass(currentParagraph.config.colWidth)"
class="paragraph-col">
<div id="{{currentParagraph.id}}_paragraphColumn"

8
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -488,7 +488,9 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
paragraphToBeFocused = note.paragraphs[index].id;
break;
}
$scope.$broadcast('updateParagraph', {paragraph: note.paragraphs[index]});
$scope.$broadcast('updateParagraph', {
note: $scope.note, // pass the note object to paragraph scope
paragraph: note.paragraphs[index]});
}
}
@ -497,7 +499,9 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
for (var idx in newParagraphIds) {
var newEntry = note.paragraphs[idx];
if (oldParagraphIds[idx] === newParagraphIds[idx]) {
$scope.$broadcast('updateParagraph', {paragraph: newEntry});
$scope.$broadcast('updateParagraph', {
note: $scope.note, // pass the note object to paragraph scope
paragraph: newEntry});
} else {
// move paragraph
var oldIdx = oldParagraphIds.indexOf(newParagraphIds[idx]);

4
zeppelin-web/src/app/notebook/notebook.html
View file

@ -84,11 +84,11 @@ limitations under the License.
<div class="note-jump"></div>
<!-- Include the paragraphs according to the note -->
<!-- Include the paragraphs according to the note, pass the note to init function -->
<div id="{{currentParagraph.id}}_paragraphColumn_main"
ng-repeat="currentParagraph in note.paragraphs"
ng-controller="ParagraphCtrl"
ng-Init="init(currentParagraph)"
ng-Init="init(currentParagraph, note)"
ng-class="columnWidthClass(currentParagraph.config.colWidth)"
class="paragraph-col">
<div class="new-paragraph" ng-click="insertNew('above')" ng-hide="viewOnly || asIframe">

19
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -18,6 +18,7 @@ angular.module('zeppelinWebApp')
.controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location,
$timeout, $compile, websocketMsgSrv) {
var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_';
$scope.parentNote = null;
$scope.paragraph = null;
$scope.originalText = '';
$scope.editor = null;
@ -28,6 +29,20 @@ angular.module('zeppelinWebApp')
$scope.compiledScope = paragraphScope;
paragraphScope.z = {
// z.runParagraph('20150213-231621_168813393')
runParagraph: function(paragraphId) {
if (paragraphId) {
var filtered = $scope.parentNote.paragraphs.filter(function(x) {
return x.id === paragraphId;});
if (filtered.length === 1) {
var paragraph = filtered[0];
websocketMsgSrv.runParagraph(paragraph.id, paragraph.title, paragraph.text,
paragraph.config, paragraph.settings.params);
} else {
// Error message here
}
}
},
// Example: z.angularBind('my_var', 'Test Value', '20150213-231621_168813393')
angularBind: function(varName, value, paragraphId) {
@ -36,6 +51,7 @@ angular.module('zeppelinWebApp')
websocketMsgSrv.clientBindAngularObject($routeParams.noteId, varName, value, paragraphId);
}
},
// Example: z.angularUnBind('my_var', '20150213-231621_168813393')
angularUnbind: function(varName, paragraphId) {
// Only push to server if paragraphId is defined
@ -55,8 +71,9 @@ angular.module('zeppelinWebApp')
};
// Controller init
$scope.init = function(newParagraph) {
$scope.init = function(newParagraph, note) {
$scope.paragraph = newParagraph;
$scope.parentNote = note;
$scope.originalText = angular.copy(newParagraph.text);
$scope.chart = {};
$scope.colWidthOption = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ];

13
zeppelin-web/src/components/navbar/navbar.controller.js
View file

@ -23,6 +23,9 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
vm.connected = websocketMsgSrv.isConnected();
vm.websocketMsgSrv = websocketMsgSrv;
vm.arrayOrderingSrv = arrayOrderingSrv;
$rootScope.fullUsername = $rootScope.ticket.principal;
$rootScope.truncatedUsername = $rootScope.ticket.principal;
var MAX_USERNAME_LENGTH=16;
angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true});
@ -43,6 +46,15 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
}
});
$scope.checkUsername = function () {
if($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
$rootScope.truncatedUsername=$rootScope.ticket.principal;
}
else {
$rootScope.truncatedUsername=$rootScope.ticket.principal.substr(0,MAX_USERNAME_LENGTH)+'..';
}
};
$scope.search = function() {
$location.url(/search/ + $scope.searchTerm);
};
@ -59,5 +71,6 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
vm.isActive = isActive;
vm.loadNotes();
$scope.checkUsername();
});

8
zeppelin-web/src/components/navbar/navbar.html
View file

@ -74,11 +74,11 @@ limitations under the License.
</span>
</div>
</form>
</li>
<li class="server-status">
<li class="server-status" >
<i class="fa fa-circle" ng-class="{'server-connected':navbar.connected, 'server-disconnected':!navbar.connected}"></i>
<span ng-show="navbar.connected">Connected</span>
<span ng-show="!navbar.connected">Disconnected</span>
<span ng-show="navbar.connected" ng-if="ticket.principal == 'anonymous' ">Connected</span>
<span ng-show="navbar.connected" ng-if="ticket.principal != 'anonymous' " tooltip-placement="bottom" tooltip="{{fullUsername}}">{{truncatedUsername}}</span>
<span ng-show="!navbar.connected">Disconnected</span>
</li>
</ul>
</div>

11
zeppelin-zengine/pom.xml
View file

@ -55,7 +55,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.10.1</version>
<version>1.10.62</version>
</dependency>
<dependency>
@ -222,4 +222,13 @@
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</build>
</project>

9
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -333,6 +333,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getBucketName() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
}
public String getEndpoint() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
}
public String getInterpreterDir() {
return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
@ -480,12 +484,15 @@ public class ZeppelinConfiguration extends XMLConfiguration {
// whether homescreen notebook will be hidden from notebook list or not
ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()),
ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner", "bin/interpreter.sh"),
ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",
System.getProperty("os.name")
.startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"),
// Decide when new note is created, interpreter settings will be binded automatically or not.
ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
View file

@ -88,10 +88,9 @@ public class AzureNotebookRepo implements NotebookRepo {
}
} catch (StorageException | URISyntaxException e) {
String msg = "Error enumerating notebooks from Azure storage";
LOG.error(msg, e);
throw new IOException(msg, e);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
}

21
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -72,14 +72,18 @@ public class S3NotebookRepo implements NotebookRepo {
// 4. Instance profile credentials delivered through the Amazon EC2 metadata service
private AmazonS3 s3client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
private static String bucketName = "";
private static String endpoint = "";
private String user = "";
private ZeppelinConfiguration conf;
public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
this.conf = conf;
user = conf.getUser();
bucketName = conf.getBucketName();
endpoint = conf.getEndpoint();
user = conf.getUser();
s3client.setEndpoint(endpoint);
}
@Override
@ -102,7 +106,17 @@ public class S3NotebookRepo implements NotebookRepo {
if (info != null) {
infos.add(info);
}
} catch (IOException e) {
} catch (AmazonServiceException ase) {
LOG.warn("Caught an AmazonServiceException for some reason.\n" +
"Error Message: {}", ase.getMessage());
} catch (AmazonClientException ace) {
LOG.info("Caught an AmazonClientException, " +
"which means the client encountered " +
"an internal error while trying to communicate" +
" with S3, " +
"such as not being able to access the network.");
LOG.info("Error Message: " + ace.getMessage());
} catch (Exception e) {
LOG.error("Can't read note ", e);
}
}
@ -110,7 +124,8 @@ public class S3NotebookRepo implements NotebookRepo {
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonServiceException ase) {
LOG.warn("Caught an AmazonServiceException for some reason.\n" +
"Error Message: {}", ase.getMessage());
} catch (AmazonClientException ace) {
LOG.info("Caught an AmazonClientException, " +
"which means the client encountered " +

10
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -60,7 +60,11 @@ public class VFSNotebookRepo implements NotebookRepo {
this.conf = conf;
try {
filesystemRoot = new URI(conf.getNotebookDir());
if (conf.isWindowsPath(conf.getNotebookDir())) {
filesystemRoot = new File(conf.getNotebookDir()).toURI();
} else {
filesystemRoot = new URI(conf.getNotebookDir());
}
} catch (URISyntaxException e1) {
throw new IOException(e1);
}
@ -72,8 +76,6 @@ public class VFSNotebookRepo implements NotebookRepo {
} catch (URISyntaxException e) {
throw new IOException(e);
}
} else {
this.filesystemRoot = filesystemRoot;
}
fsManager = VFS.getManager();
@ -134,7 +136,7 @@ public class VFSNotebookRepo implements NotebookRepo {
if (info != null) {
infos.add(info);
}
} catch (IOException e) {
} catch (Exception e) {
logger.error("Can't read note " + f.getName().toString(), e);
}
}

28
zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java
View file

@ -17,14 +17,32 @@
package org.apache.zeppelin.util;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
/**
* TODO(moon) : add description.
*/
public class Util {
private static final String PROJECT_PROPERTIES_VERSION_KEY = "version";
private static Properties projectProperties;
static {
projectProperties = new Properties();
try {
projectProperties.load(Util.class.getResourceAsStream("/project.properties"));
} catch (IOException e) {
//Fail to read project.properties
}
}
public static String[] split(String str, char split) {
return split(str, new String[] {String.valueOf(split)}, false);
@ -181,4 +199,14 @@ public class Util {
return false;
}
}
/**
* Get Zeppelin version
*
* @return Current Zeppelin version
*/
public static String getVersion() {
return StringUtils.defaultIfEmpty(projectProperties.getProperty(PROJECT_PROPERTIES_VERSION_KEY),
StringUtils.EMPTY);
}
}

19
zeppelin-zengine/src/main/resources/project.properties Normal file
View file

@ -0,0 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version=${project.version}
artifactId=${project.artifactId}

14
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -90,6 +90,20 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
}
}
@Test
public void testInvalidJsonFile() throws IOException {
// given
int numNotes = notebookRepo.list().size();
// when create invalid json file
File testNoteDir = new File(mainNotebookDir, "test");
testNoteDir.mkdir();
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
// then
assertEquals(numNotes, notebookRepo.list().size());
}
@Test
public void testSaveNotebook() throws IOException, InterruptedException {
Note note = notebook.createNote();

6
zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java
View file

@ -97,4 +97,10 @@ public class UtilTest extends TestCase {
assertEquals("array <STRUCT<STRING>> tags", t[0]);
assertEquals("aa", t[2]);
}
public void testGetVersion(){
String version = Util.getVersion();
assertNotNull(version);
System.out.println(version);
}
}