Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/master' into ZEPPELIN-1999

Browse source
This commit is contained in:
Tinkoff DWH 2017-03-21 00:28:46 +05:00
commit 93c759d754
131 changed files with 4610 additions and 2452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -100,6 +100,9 @@ Thumbs.db
target/
**/target/
# maven flattened pom files
**/.flattened-pom.xml
# Generated by Jekyll
docs/_site/

52
.travis.yml
View file

@ -25,7 +25,6 @@ cache:
- ${HOME}/R
- zeppelin-web/node
- zeppelin-web/node_modules
- zeppelin-web/bower_components
addons:
apt:
@ -34,41 +33,52 @@ addons:
packages:
- r-base-dev
env:
global:
# Interpreters does not required by zeppelin-server integration tests
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy'
matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.1.0 and scala 2.11
- sudo: required
jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with spark 2.0.2 and scala 2.11
# Test core modules
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test spark module for 1.6.3 with scala 2.10
- sudo: required
jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10 -Plivy-0.2" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11 -Dscala.version=2.11.7 -Dscala.binary.version=2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
# Test interpreter modules
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
# Test spark module for 2.1.0 with scala 2.11, livy
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
# Test spark module for 2.0.2 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test python/pyspark with python 2
- jdk: "oraclejdk7"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
# Test python/pyspark with python 3
- jdk: "oraclejdk7"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
before_install:
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
@ -83,7 +93,7 @@ install:
- mvn $BUILD_FLAG $MODULES $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
- if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-server-$LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi

10
bin/interpreter.sh
View file

@ -20,10 +20,10 @@ bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
function usage() {
echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
}
while getopts "hp:d:l:v:u:" o; do
while getopts "hp:d:l:v:u:n:" o; do
case ${o} in
h)
usage
@ -50,6 +50,9 @@ while getopts "hp:d:l:v:u:" o; do
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
;;
g)
INTERPRETER_GROUP_NAME=${OPTARG}
;;
esac
done
@ -86,6 +89,9 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then
ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-"
fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
fi

1
conf/zeppelin-env.cmd.template
View file

@ -38,6 +38,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_SSE REM Server-side encryption enable
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?

1
conf/zeppelin-env.sh.template
View file

@ -43,6 +43,7 @@
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise

12
conf/zeppelin-site.xml.template
View file

@ -239,6 +239,18 @@
<description>Local repository for interpreter's additional dependency loading</description>
</property>
<property>
<name>zeppelin.interpreter.dep.mvnRepo</name>
<value>http://repo1.maven.org/maven2/</value>
<description>Remote principal repository for interpreter's additional dependency loading</description>
</property>
<property>
<name>zeppelin.dep.localrepo</name>
<value>local-repo</value>
<description>Local repository for dependency loader</description>
</property>
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>

24
dev/change_scala_version.sh
View file

@ -34,7 +34,7 @@ if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then
usage
fi
TO_VERSION=$1
TO_VERSION="$1"
check_scala_version() {
for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
@ -42,12 +42,14 @@ check_scala_version() {
exit 1
}
check_scala_version "$TO_VERSION"
check_scala_version "${TO_VERSION}"
if [ $TO_VERSION = "2.11" ]; then
if [ "${TO_VERSION}" = "2.11" ]; then
FROM_VERSION="2.10"
SCALA_LIB_VERSION="2.11.7"
else
FROM_VERSION="2.11"
SCALA_LIB_VERSION="2.10.5"
fi
sed_i() {
@ -57,11 +59,17 @@ sed_i() {
export -f sed_i
BASEDIR=$(dirname $0)/..
find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \;
find "${BASEDIR}" -name 'pom.xml' -not -path '*target*' -print \
-exec bash -c "sed_i 's/\(artifactId.*\)_'${FROM_VERSION}'/\1_'${TO_VERSION}'/g' {}" \;
# Also update <scala.binary.version> in parent POM
# update <scala.binary.version> in parent POM
# Match any scala binary version to ensure idempotency
sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'$TO_VERSION'</' \
"$BASEDIR/pom.xml"
sed_i '1,/<scala\.binary\.version>[0-9]*\.[0-9]*</s/<scala\.binary\.version>[0-9]*\.[0-9]*</<scala.binary.version>'${TO_VERSION}'</' \
"${BASEDIR}/pom.xml"
# update <scala.version> in parent POM
# This is to make variables in leaf pom to be substituted to real value when flattened-pom is created.
# maven-flatten plugin doesn't take properties defined under profile even if scala-2.11/scala-2.10 is activated via -Pscala-2.11/-Pscala-2.10,
# and use default defined properties to create flatten pom.
sed_i '1,/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</s/<scala\.version>[0-9]*\.[0-9]*\.[0-9]*</<scala.version>'${SCALA_LIB_VERSION}'</' \
"${BASEDIR}/pom.xml"

BIN
docs/assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 9.2 KiB

2
docs/install/build.md
View file

@ -210,7 +210,7 @@ mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
Ignite Interpreter
```bash
mvn clean package -Dignite.version=1.8.0 -DskipTests
mvn clean package -Dignite.version=1.9.0 -DskipTests
```
Scalding Interpreter

20
docs/install/configuration.md
View file

@ -26,7 +26,7 @@ limitations under the License.
## Zeppelin Properties
There are two locations you can configure Apache Zeppelin.
* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
* **Java properties** can ba defined in `conf/zeppelin-site.xml`.
If both are defined, then the **environment variables** will take priority.
@ -43,7 +43,9 @@ If both are defined, then the **environment variables** will take priority.
<td><h6 class="properties">ZEPPELIN_PORT</h6></td>
<td><h6 class="properties">zeppelin.server.port</h6></td>
<td>8080</td>
<td>Zeppelin server port</td>
<td>Zeppelin server port </br>
<span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with
<a href="https://zeppelin.apache.org/contribution/webapplication.html#dev-mode" target="_blank">Zeppelin web application development port</a> (default: 9000).</span></td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_SSL_PORT</h6></td>
@ -255,6 +257,12 @@ If both are defined, then the **environment variables** will take priority.
<td>interpreter</td>
<td>Interpreter directory</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_INTERPRETER_DEP_MVNREPO</h6></td>
<td><h6 class="properties">zeppelin.interpreter.dep.mvnRepo</h6></td>
<td>http://repo1.maven.org/maven2/</td>
<td>Remote principal repository for interpreter's additional dependency loading</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_INTERPRETER_OUTPUT_LIMIT</h6></td>
<td><h6 class="properties">zeppelin.interpreter.output.limit</h6></td>
@ -267,6 +275,12 @@ If both are defined, then the **environment variables** will take priority.
<td>30000</td>
<td>Output message from interpreter exceeding the limit will be truncated</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_DEP_LOCALREPO</h6></td>
<td><h6 class="properties">zeppelin.dep.localrepo</h6></td>
<td>local-repo</td>
<td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td>
<td><h6 class="properties">zeppelin.websocket.max.text.message.size</h6></td>
@ -383,7 +397,7 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order
### Obfuscating Passwords using the Jetty Password Tool
Security best practices advise to not use plain text passwords and Jetty provides a password tool to help obfuscating the passwords used to access the KeyStore and TrustStore.
The Password tool documentation can be found [here](http://www.eclipse.org/jetty/documentation/current/configuring-security-secure-passwords.html).
After using the tool:

5
docs/install/upgrade.md
View file

@ -56,3 +56,8 @@ So, copying `notebook` and `conf` directory should be enough.
- From 0.7, we uses `pegdown` as the `markdown.parser.type` option for the `%md` interpreter. Rendered markdown might be different from what you expected
- From 0.7 note.json format has been changed to support multiple outputs in a paragraph. Zeppelin will automatically convert old format to new format. 0.6 or lower version can read new note.json format but output will not be displayed. For the detail, see [ZEPPELIN-212](http://issues.apache.org/jira/browse/ZEPPELIN-212) and [pull request](https://github.com/apache/zeppelin/pull/1658).
- From 0.7 note storage layer will utilize `GitNotebookRepo` by default instead of `VFSNotebookRepo` storage layer, which is an extension of latter one with versioning capabilities on top of it.
### Upgrading from Zeppelin 0.7 to 0.8
- From 0.8, we recommend to use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` instead of `zeppelin.pyspark.python` as `zeppelin.pyspark.python` only effects driver. You can use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` as using them in spark.

79
docs/interpreter/jdbc.md
View file

@ -118,6 +118,11 @@ The JDBC interpreter properties are defined by default like below.
<td>gpadmin</td>
<td>The JDBC user name</td>
</tr>
<tr>
<td>default.precode</td>
<td></td>
<td>Some SQL which executes while opening connection</td>
</tr>
</table>
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
@ -217,6 +222,75 @@ SELECT name, country, performer
FROM demo.performers
WHERE name='{{"{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia"}}}}'
```
### Usage *precode*
You can set *precode* for each data source. Code runs once while opening the connection.
##### Properties
An example settings of interpreter for the two data sources, each of which has its *precode* parameter.
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Value</th>
</tr>
<tr>
<td>default.driver</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>1</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:postgresql://localhost:5432/</td>
</tr>
<tr>
<td>default.user</td>
<td>postgres</td>
</tr>
<tr>
<td>default.precode</td>
<td>set search_path='test_path'</td>
</tr>
<tr>
<td>mysql.driver</td>
<td>com.mysql.jdbc.Driver</td>
</tr>
<tr>
<td>mysql.password</td>
<td>1</td>
</tr>
<tr>
<td>mysql.url</td>
<td>jdbc:mysql://localhost:3306/</td>
</tr>
<tr>
<td>mysql.user</td>
<td>root</td>
</tr>
<tr>
<td>mysql.precode</td>
<td>set @v=12</td>
</tr>
</table>
##### Usage
Test of execution *precode* for each data source.
```sql
%jdbc
show search_path
```
Returns value of `search_path` which is set in the *default.precode*.
```sql
%jdbc(mysql)
select @v
```
Returns value of `v` which is set in the *mysql.precode*.
## Examples
Here are some examples you can refer to. Including the below connectors, you can connect every databases as long as it can be configured with it's JDBC driver.
@ -423,8 +497,13 @@ Here are some examples you can refer to. Including the below connectors, you can
<td>default.password</td>
<td>hive_password</td>
</tr>
<tr>
<td>hive.proxy.user</td>
<td>true or false</td>
</table>
Connection to Hive JDBC with a proxy user can be disabled with `hive.proxy.user` property (set to true by default)
[Apache Hive 1 JDBC Driver Docs](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC)
[Apache Hive 2 JDBC Driver Docs](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC)

11
docs/interpreter/spark.md
View file

@ -104,9 +104,16 @@ You can also set other Spark properties which are not listed in the table. For a
<td>Local repository for dependency loader</td>
</tr>
<tr>
<td>zeppelin.pyspark.python</td>
<td>PYSPARK_PYTHON</td>
<td>python</td>
<td>Python command to run pyspark with</td>
<td>Python binary executable to use for PySpark in both driver and workers (default is <code>python</code>).
Property <code>spark.pyspark.python</code> take precedence if it is set</td>
</tr>
<tr>
<td>PYSPARK_DRIVER_PYTHON</td>
<td>python</td>
<td>Python binary executable to use for PySpark in driver only (default is <code>PYSPARK_PYTHON</code>).
Property <code>spark.pyspark.driver.python</code> take precedence if it is set</td>
</tr>
<tr>
<td>zeppelin.spark.concurrentSQL</td>

16
docs/manual/dynamicform.md
View file

@ -56,7 +56,13 @@ Also you can separate option's display name and value, using `${formName=default
<img src="../assets/themes/zeppelin/img/screenshots/form_select_displayname.png" />
Hit enter after selecting option to run the paragraph with new value.
The paragraph will be automatically run after you change your selection by default.
But in case you have multiple types dynamic form in one paragraph, you might want to run the paragraph after changing all the selections.
You can control this by unchecking the below **Run on selection change** option in the setting menu.
Even if you uncheck this option, still you can run it by pressing `Enter`.
<img src="../assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png" />
### Checkbox form
@ -68,6 +74,14 @@ Besides, you can specify the delimiter using `${checkbox(delimiter):formName=...
<img src="../assets/themes/zeppelin/img/screenshots/form_checkbox_delimiter.png">
Like [select form](#select-form), the paragraph will be automatically run after you change your selection by default.
But in case you have multiple types dynamic form in one paragraph, you might want to run the paragraph after changing all the selections.
You can control this by unchecking the below **Run on selection change** option in the setting menu.
Even if you uncheck this option, still you can run it by pressing `Enter`.
<img src="../assets/themes/zeppelin/img/screenshots/selectForm-checkbox.png" />
## Creates Programmatically
Some language backend uses programmatic way to create form. For example [ZeppelinContext](../interpreter/spark.html#zeppelincontext) provides form creation API

4
docs/manual/interpreterinstallation.md
View file

@ -111,9 +111,9 @@ You can also install 3rd party interpreters located in the maven repository by u
./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1
```
The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
The above command will download maven artifact `groupId1:artifact1:version1` and all of its transitive dependencies into `interpreter/interpreter1` directory.
After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your note](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
#### Install multiple 3rd party interpreters at once

2
ignite/pom.xml
View file

@ -32,7 +32,7 @@
<name>Zeppelin: Apache Ignite interpreter</name>
<properties>
<ignite.version>1.8.0</ignite.version>
<ignite.version>1.9.0</ignite.version>
</properties>
<dependencies>

6
interpreter/lib/python/backend_zinline.py
View file

@ -18,6 +18,7 @@
from __future__ import print_function
import sys
import uuid
import warnings
import base64
@ -94,7 +95,10 @@ class FigureCanvasZInline(FigureCanvasAgg):
buf = BytesIO()
self.print_figure(buf, **kwargs)
fmt = fmt.encode()
byte_str = b"data:image/%s;base64," %fmt
if sys.version_info >= (3, 4) and sys.version_info < (3, 5):
byte_str = bytes("data:image/%s;base64," %fmt, "utf-8")
else:
byte_str = b"data:image/%s;base64," %fmt
byte_str += base64.b64encode(buf.getvalue())
# Python3 forces all strings to default to unicode, but for raster image

59
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -14,14 +14,7 @@
*/
package org.apache.zeppelin.jdbc;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import static org.apache.commons.lang.StringUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
import java.sql.DriverManager;
@ -37,11 +30,11 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import com.google.common.base.Throwables;
import org.apache.commons.dbcp2.ConnectionFactory;
import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
@ -49,7 +42,10 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.jdbc.security.JDBCSecurityImpl;
@ -61,9 +57,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import static org.apache.commons.lang.StringUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
/**
* JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ,
@ -101,8 +101,10 @@ public class JDBCInterpreter extends Interpreter {
static final String URL_KEY = "url";
static final String USER_KEY = "user";
static final String PASSWORD_KEY = "password";
static final String PRECODE_KEY = "precode";
static final String JDBC_JCEKS_FILE = "jceks.file";
static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
static final String PRECODE_KEY_TEMPLATE = "%s.precode";
static final String DOT = ".";
private static final char WHITESPACE = ' ';
@ -117,6 +119,7 @@ public class JDBCInterpreter extends Interpreter {
static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
static final String DEFAULT_PRECODE = DEFAULT_KEY + DOT + PRECODE_KEY;
static final String EMPTY_COLUMN_VALUE = "";
@ -340,6 +343,9 @@ public class JDBCInterpreter extends Interpreter {
if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
createConnectionPool(url, user, propertyKey, properties);
try (Connection connection = DriverManager.getConnection(jdbcDriver)) {
executePrecode(connection, propertyKey);
}
}
return DriverManager.getConnection(jdbcDriver);
}
@ -374,16 +380,20 @@ public class JDBCInterpreter extends Interpreter {
if (lastIndexOfUrl == -1) {
lastIndexOfUrl = connectionUrl.length();
}
connectionUrl.insert(lastIndexOfUrl, ";hive.server2.proxy.user=" + user + ";");
boolean hasProxyUser = property.containsKey("hive.proxy.user");
if (!hasProxyUser || !property.getProperty("hive.proxy.user").equals("false")){
logger.debug("Using hive proxy user");
connectionUrl.insert(lastIndexOfUrl, ";hive.server2.proxy.user=" + user + ";");
}
connection = getConnectionFromPool(connectionUrl.toString(),
user, propertyKey, properties);
user, propertyKey, properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
ugi = UserGroupInformation.createProxyUser(
user, UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
logger.error("Error in getCurrentUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
@ -540,6 +550,20 @@ public class JDBCInterpreter extends Interpreter {
return queries;
}
private void executePrecode(Connection connection, String propertyKey) throws SQLException {
String precode = getProperty(String.format(PRECODE_KEY_TEMPLATE, propertyKey));
if (StringUtils.isNotBlank(precode)) {
precode = StringUtils.trim(precode);
logger.info("Run SQL precode '{}'", precode);
try (Statement statement = connection.createStatement()) {
statement.execute(precode);
if (!connection.getAutoCommit()) {
connection.commit();
}
}
}
}
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
Connection connection;
@ -611,7 +635,7 @@ public class JDBCInterpreter extends Interpreter {
} catch (SQLException e) { /*ignored*/ }
}
getJDBCConfiguration(user).removeStatement(paragraphId);
} catch (Exception e) {
} catch (Throwable e) {
if (e.getCause() instanceof TTransportException &&
Throwables.getStackTraceAsString(e).contains("GSS") &&
getJDBCConfiguration(user).isConnectionInDBDriverPoolSuccessful(propertyKey)) {
@ -761,4 +785,3 @@ public class JDBCInterpreter extends Interpreter {
}
}
}

2
jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
View file

@ -111,6 +111,8 @@ public class SqlCompleter extends StringsCompleter {
try {
while (schemas.next()) {
String schemaName = schemas.getString("TABLE_SCHEM");
if (schemaName == null)
schemaName = "";
if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
schemaFilter.replace("_", ".").replace("%", ".*?"))) {
res.add(schemaName);

6
jdbc/src/main/resources/interpreter-setting.json
View file

@ -28,6 +28,12 @@
"defaultValue": "org.postgresql.Driver",
"description": "JDBC Driver Name"
},
"default.precode": {
"envName": null,
"propertyName": "zeppelin.jdbc.precode",
"defaultValue": "",
"description": "SQL which executes while opening connection"
},
"common.max_count": {
"envName": null,
"propertyName": "common.max_count",

64
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -19,6 +19,8 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.junit.Assert.*;
@ -43,6 +45,7 @@ import org.junit.Before;
import org.junit.Test;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
/**
* JDBC interpreter unit tests
*/
@ -386,4 +389,63 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertNull(user2JDBC2Conf.getPropertyMap("default").get("password"));
jdbc2.close();
}
}
@Test
public void testPrecode() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty(DEFAULT_PRECODE, "SET @testVariable=1");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
String sqlQuery = "select @testVariable";
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("@TESTVARIABLE\n1\n", interpreterResult.message().get(0).getData());
}
@Test
public void testIncorrectPrecode() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty(DEFAULT_PRECODE, "incorrect command");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
String sqlQuery = "select 1";
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType());
}
@Test
public void testPrecodeWithAnotherPrefix() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("anotherPrefix.driver", "org.h2.Driver");
properties.setProperty("anotherPrefix.url", getJdbcConnection());
properties.setProperty("anotherPrefix.user", "");
properties.setProperty("anotherPrefix.password", "");
properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "SET @testVariable=2");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
String sqlQuery = "(anotherPrefix) select @testVariable";
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
}
}

1095
livy/pom.xml
View file

File diff suppressed because it is too large Load diff

2
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
View file

@ -55,7 +55,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
private int pullStatusInterval;
protected boolean displayAppInfo;
private AtomicBoolean sessionExpired = new AtomicBoolean(false);
private LivyVersion livyVersion;
protected LivyVersion livyVersion;
// keep tracking the mapping between paragraphId and statementId, so that we can cancel the
// statement after we execute it.

85
livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
View file

@ -82,13 +82,13 @@ public class LivyInterpreterIT {
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
final LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
"title", "text", authInfo, null, null, null, null, null, output);
sparkInterpreter.open();
@ -158,6 +158,31 @@ public class LivyInterpreterIT {
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("incomplete statement"));
// cancel
if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 3 seconds to wait job starting
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkInterpreter
.interpret("sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
}
} finally {
sparkInterpreter.close();
}
@ -289,11 +314,11 @@ public class LivyInterpreterIT {
return;
}
LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties);
final LivyPySparkInterpreter pysparkInterpreter = new LivyPySparkInterpreter(properties);
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.pyspark",
final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.pyspark",
"title", "text", authInfo, null, null, null, null, null, output);
pysparkInterpreter.open();
@ -341,6 +366,31 @@ public class LivyInterpreterIT {
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("name 'a' is not defined"));
// cancel
if (pysparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 3 seconds to wait job starting
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
pysparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = pysparkInterpreter
.interpret("import time\n" +
"sc.range(1, 10).foreach(lambda a: time.sleep(10))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
}
} finally {
pysparkInterpreter.close();
}
@ -384,7 +434,7 @@ public class LivyInterpreterIT {
return;
}
LivySparkRInterpreter sparkRInterpreter = new LivySparkRInterpreter(properties);
final LivySparkRInterpreter sparkRInterpreter = new LivySparkRInterpreter(properties);
try {
sparkRInterpreter.getLivyVersion();
} catch (APINotFoundException e) {
@ -394,7 +444,7 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sparkr",
final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.sparkr",
"title", "text", authInfo, null, null, null, null, null, output);
sparkRInterpreter.open();
@ -408,6 +458,29 @@ public class LivyInterpreterIT {
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
// cancel
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 3 seconds to wait job starting
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkRInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\n" +
"df1 <- dapplyCollect(df, function(x) " +
"{ Sys.sleep(10); x <- cbind(x, x$waiting * 60) })", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
} else {
result = sparkRInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)" +
"\nhead(df)", context);

89
notebook/2A94M5J1Z/note.json
View file

@ -32,9 +32,6 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423836981412_-1007008116",
"id": "20150213-231621_168813393",
"results": {
"code": "SUCCESS",
"msg": [
@ -44,6 +41,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423836981412_-1007008116",
"id": "20150213-231621_168813393",
"dateCreated": "Feb 13, 2015 11:16:21 PM",
"dateStarted": "Dec 17, 2016 3:32:15 PM",
"dateFinished": "Dec 17, 2016 3:32:18 PM",
@ -78,9 +78,6 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423500779206_-1502780787",
"id": "20150210-015259_1403135953",
"results": {
"code": "SUCCESS",
"msg": [
@ -90,6 +87,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423500779206_-1502780787",
"id": "20150210-015259_1403135953",
"dateCreated": "Feb 10, 2015 1:52:59 AM",
"dateStarted": "Dec 17, 2016 3:30:09 PM",
"dateFinished": "Dec 17, 2016 3:30:58 PM",
@ -99,16 +99,17 @@
{
"text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:13 PM",
"dateUpdated": "Mar 17, 2017 12:18:02 PM",
"config": {
"colWidth": 4.0,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"mode": "multiBarChart",
"height": 366.0,
"optionOpen": false
}
},
"helium": {}
}
],
"enabled": true,
@ -122,9 +123,6 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423500782552_-1439281894",
"id": "20150210-015302_1492795503",
"results": {
"code": "SUCCESS",
"msg": [
@ -134,6 +132,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423500782552_-1439281894",
"id": "20150210-015302_1492795503",
"dateCreated": "Feb 10, 2015 1:53:02 AM",
"dateStarted": "Dec 17, 2016 3:30:13 PM",
"dateFinished": "Dec 17, 2016 3:31:04 PM",
@ -143,16 +144,17 @@
{
"text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:16 PM",
"dateUpdated": "Mar 17, 2017 12:17:39 PM",
"config": {
"colWidth": 4.0,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"mode": "multiBarChart",
"height": 294.0,
"optionOpen": false
}
},
"helium": {}
}
],
"enabled": true,
@ -174,9 +176,6 @@
}
}
},
"apps": [],
"jobName": "paragraph_1423720444030_-1424110477",
"id": "20150212-145404_867439529",
"results": {
"code": "SUCCESS",
"msg": [
@ -186,6 +185,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423720444030_-1424110477",
"id": "20150212-145404_867439529",
"dateCreated": "Feb 12, 2015 2:54:04 PM",
"dateStarted": "Dec 17, 2016 3:30:58 PM",
"dateFinished": "Dec 17, 2016 3:31:07 PM",
@ -195,16 +197,17 @@
{
"text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
"user": "anonymous",
"dateUpdated": "Dec 17, 2016 3:30:18 PM",
"dateUpdated": "Mar 17, 2017 12:18:18 PM",
"config": {
"colWidth": 4.0,
"results": [
{
"graph": {
"mode": "table",
"height": 300.0,
"mode": "stackedAreaChart",
"height": 280.0,
"optionOpen": false
}
},
"helium": {}
}
],
"enabled": true,
@ -237,9 +240,6 @@
}
}
},
"apps": [],
"jobName": "paragraph_1423836262027_-210588283",
"id": "20150213-230422_1600658137",
"results": {
"code": "SUCCESS",
"msg": [
@ -249,6 +249,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423836262027_-210588283",
"id": "20150213-230422_1600658137",
"dateCreated": "Feb 13, 2015 11:04:22 PM",
"dateStarted": "Dec 17, 2016 3:31:05 PM",
"dateFinished": "Dec 17, 2016 3:31:09 PM",
@ -283,9 +286,6 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1423836268492_216498320",
"id": "20150213-230428_1231780373",
"results": {
"code": "SUCCESS",
"msg": [
@ -295,6 +295,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1423836268492_216498320",
"id": "20150213-230428_1231780373",
"dateCreated": "Feb 13, 2015 11:04:28 PM",
"dateStarted": "Dec 17, 2016 3:30:24 PM",
"dateFinished": "Dec 17, 2016 3:30:29 PM",
@ -329,9 +332,6 @@
"params": {},
"forms": {}
},
"apps": [],
"jobName": "paragraph_1427420818407_872443482",
"id": "20150326-214658_12335843",
"results": {
"code": "SUCCESS",
"msg": [
@ -341,6 +341,9 @@
}
]
},
"apps": [],
"jobName": "paragraph_1427420818407_872443482",
"id": "20150326-214658_12335843",
"dateCreated": "Mar 26, 2015 9:46:58 PM",
"dateStarted": "Dec 17, 2016 3:30:34 PM",
"dateFinished": "Dec 17, 2016 3:30:34 PM",
@ -364,25 +367,7 @@
"name": "Zeppelin Tutorial/Basic Features (Spark)",
"id": "2A94M5J1Z",
"angularObjects": {
"2C6WUGPNH:shared_process": [],
"2C4A8RJNB:shared_process": [],
"2C4DTK2ZT:shared_process": [],
"2C6XKJWBR:shared_process": [],
"2C6AHZPMK:shared_process": [],
"2C5SU66WQ:shared_process": [],
"2C6AMJ98Q:shared_process": [],
"2C4AJZK72:shared_process": [],
"2C3STPSD7:shared_process": [],
"2C4FJN9CK:shared_process": [],
"2C3CW6JBY:shared_process": [],
"2C5UPQX6Q:shared_process": [],
"2C5873KN4:shared_process": [],
"2C5719XN4:shared_process": [],
"2C52DE5G3:shared_process": [],
"2C4G28E63:shared_process": [],
"2C6CU96BC:shared_process": [],
"2C49A6WY3:shared_process": [],
"2C3NE73HG:shared_process": []
"2C73DY9P9:shared_process": []
},
"config": {
"looknfeel": "default"

93
pom.xml
View file

@ -308,6 +308,32 @@
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.0.0</version>
<configuration>
<flattenMode>ossrh</flattenMode>
<updatePomFile>true</updatePomFile>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Test coverage plugin -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
@ -475,7 +501,7 @@
<version>${plugin.deploy.version}</version>
</plugin>
<!--TODO(alex): make part of the build and reconcile conflicts
<!--TODO(alex): make part of the build and reconcile conflicts
<plugin>
<groupId>com.ning.maven.plugins</groupId>
<artifactId>maven-duplicate-finder-plugin</artifactId>
@ -898,6 +924,39 @@
<exclude>**/interpreter-setting.json</exclude>
<exclude>**/constants.json</exclude>
<exclude>scripts/**</exclude>
<exclude>**/**/*.log</exclude>
<exclude>**/**/logs/**</exclude>
<!-- bundled from zeppelin-web -->
<exclude>**/test/karma.conf.js</exclude>
<exclude>**/test/spec/**</exclude>
<exclude>**/.babelrc</exclude>
<exclude>**/.bowerrc</exclude>
<exclude>.editorconfig</exclude>
<exclude>.eslintrc</exclude>
<exclude>**/.tmp/**</exclude>
<exclude>**/target/**</exclude>
<exclude>**/node/**</exclude>
<exclude>**/node_modules/**</exclude>
<exclude>**/bower_components/**</exclude>
<exclude>**/dist/**</exclude>
<exclude>**/.buildignore</exclude>
<exclude>**/.npmignore</exclude>
<exclude>**/.jshintrc</exclude>
<exclude>**/yarn.lock</exclude>
<exclude>**/bower.json</exclude>
<exclude>**/src/fonts/Patua-One*</exclude>
<exclude>**/src/fonts/patua-one*</exclude>
<exclude>**/src/fonts/Roboto*</exclude>
<exclude>**/src/fonts/roboto*</exclude>
<exclude>**/src/fonts/fontawesome*</exclude>
<exclude>**/src/fonts/font-awesome*</exclude>
<exclude>**/src/styles/font-awesome*</exclude>
<exclude>**/src/fonts/Simple-Line*</exclude>
<exclude>**/src/fonts/simple-line*</exclude>
<exclude>**/src/fonts/Source-Code-Pro*</exclude>
<exclude>**/src/fonts/source-code-pro*</exclude>
<exclude>**/src/**/**.test.js</exclude>
<!-- from SQLLine 1.0.2, see ZEPPELIN-2135 -->
<exclude>**/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java</exclude>
@ -947,22 +1006,24 @@
<exclude>**/package.json</exclude>
<!-- compiled R packages (binaries) -->
<exclude>R/lib/**</exclude>
<exclude>r/lib/**</exclude>
<exclude>**/R/lib/**</exclude>
<exclude>**/r/lib/**</exclude>
<!--R-related files with alternative licenses-->
<!--R-related files with alternative licenses-->
<exclude>r/R/rzeppelin/R/globals.R</exclude>
<exclude>r/R/rzeppelin/R/common.R</exclude>
<exclude>r/R/rzeppelin/R/protocol.R</exclude>
<exclude>r/R/rzeppelin/R/rServer.R</exclude>
<exclude>r/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>r/R/rzeppelin/R/zzz.R</exclude>
<exclude>r/src/main/scala/scala/Console.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>r/R/rzeppelin/DESCRIPTION</exclude>
<exclude>r/R/rzeppelin/NAMESPACE</exclude>
<exclude>**/R/rzeppelin/R/globals.R</exclude>
<exclude>**/R/rzeppelin/R/common.R</exclude>
<exclude>**/R/rzeppelin/R/protocol.R</exclude>
<exclude>**/R/rzeppelin/R/rServer.R</exclude>
<exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>**/R/rzeppelin/R/zzz.R</exclude>
<exclude>**/src/main/scala/scala/Console.scala</exclude>
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--The following files are mechanical-->
<exclude>**/R/rzeppelin/DESCRIPTION</exclude>
<exclude>**/R/rzeppelin/NAMESPACE</exclude>
</excludes>
</configuration>

37
python/pom.xml
View file

@ -98,6 +98,43 @@
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>wagon-maven-plugin</artifactId>
<version>1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals><goal>download-single</goal></goals>
<configuration>
<url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
<fromFile>py4j-${py4j.version}.zip</fromFile>
<toFile>${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip</toFile>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<phase>package</phase>
<configuration>
<target>
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip"
dest="${project.build.directory}/../../interpreter/python"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>

38
python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java
View file

@ -21,10 +21,8 @@ import org.apache.zeppelin.scheduler.Scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.*;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -37,6 +35,7 @@ public class PythonDockerInterpreter extends Interpreter {
Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
Pattern deactivatePattern = Pattern.compile("deactivate");
Pattern helpPattern = Pattern.compile("help");
private File zeppelinHome;
public PythonDockerInterpreter(Properties property) {
super(property);
@ -44,7 +43,11 @@ public class PythonDockerInterpreter extends Interpreter {
@Override
public void open() {
if (System.getenv("ZEPPELIN_HOME") != null) {
zeppelinHome = new File(System.getenv("ZEPPELIN_HOME"));
} else {
zeppelinHome = Paths.get("..").toAbsolutePath().toFile();
}
}
@Override
@ -54,6 +57,7 @@ public class PythonDockerInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
File pythonScript = new File(getPythonInterpreter().getScriptPath());
InterpreterOutput out = context.out;
Matcher activateMatcher = activatePattern.matcher(st);
@ -66,7 +70,28 @@ public class PythonDockerInterpreter extends Interpreter {
} else if (activateMatcher.matches()) {
String image = activateMatcher.group(1);
pull(out, image);
setPythonCommand("docker run -i --rm " + image + " python -iu");
// mount pythonscript dir
String mountPythonScript = "-v " +
pythonScript.getParentFile().getAbsolutePath() +
":/_zeppelin_tmp ";
// mount zeppelin dir
String mountPy4j = "-v " +
zeppelinHome.getAbsolutePath() +
":/_zeppelin ";
// set PYTHONPATH
String pythonPath = ":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PY4JPATH + ":" +
":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PYTHON_LIBS;
setPythonCommand("docker run -i --rm " +
mountPythonScript +
mountPy4j +
"-e PYTHONPATH=\"" + pythonPath + "\" " +
image + " " +
getPythonInterpreter().getPythonBindPath() + " " +
"/_zeppelin_tmp/" + pythonScript.getName());
restartPythonProcess();
out.clear();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated");
@ -79,6 +104,7 @@ public class PythonDockerInterpreter extends Interpreter {
}
}
public void setPythonCommand(String cmd) {
PythonInterpreter python = getPythonInterpreter();
python.setPythonCommand(cmd);

500
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -18,20 +18,38 @@
package org.apache.zeppelin.python;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.ServerSocket;
import java.io.OutputStreamWriter;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.net.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.exec.ExecuteResultHandler;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.util.InterpreterOutputStream;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -39,32 +57,164 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import py4j.GatewayServer;
import py4j.commands.Command;
/**
* Python interpreter for Zeppelin.
*/
public class PythonInterpreter extends Interpreter {
public class PythonInterpreter extends Interpreter implements ExecuteResultHandler {
private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreter.class);
public static final String BOOTSTRAP_PY = "/bootstrap.py";
public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py";
public static final String ZEPPELIN_PYTHON = "zeppelin.python";
public static final String ZEPPELIN_PYTHON = "python/zeppelin_python.py";
public static final String ZEPPELIN_PY4JPATH = "interpreter/python/py4j-0.9.2/src";
public static final String ZEPPELIN_PYTHON_LIBS = "interpreter/lib/python";
public static final String DEFAULT_ZEPPELIN_PYTHON = "python";
public static final String MAX_RESULT = "zeppelin.python.maxResult";
private Integer port;
private GatewayServer gatewayServer;
private Boolean py4JisInstalled = false;
private InterpreterContext context;
private Pattern errorInLastLine = Pattern.compile(".*(Error|Exception): .*$");
private String pythonPath;
private int maxResult;
private String py4jLibPath;
private String pythonLibPath;
PythonProcess process = null;
private String pythonCommand = null;
private String pythonCommand;
private GatewayServer gatewayServer;
private DefaultExecutor executor;
private int port;
private InterpreterOutputStream outputStream;
private BufferedWriter ins;
private PipedInputStream in;
private ByteArrayOutputStream input;
private String scriptPath;
boolean pythonscriptRunning = false;
private static final int MAX_TIMEOUT_SEC = 10;
private long pythonPid = 0;
Integer statementSetNotifier = new Integer(0);
public PythonInterpreter(Properties property) {
super(property);
try {
File scriptFile = File.createTempFile("zeppelin_python-", ".py", new File("/tmp"));
scriptPath = scriptFile.getAbsolutePath();
} catch (IOException e) {
throw new InterpreterException(e);
}
}
private String workingDir() {
URL myURL = getClass().getProtectionDomain().getCodeSource().getLocation();
java.net.URI myURI = null;
try {
myURI = myURL.toURI();
} catch (URISyntaxException e1)
{}
String path = java.nio.file.Paths.get(myURI).toFile().toString();
return path;
}
private void createPythonScript() {
File out = new File(scriptPath);
if (out.exists() && out.isDirectory()) {
throw new InterpreterException("Can't create python script " + out.getAbsolutePath());
}
copyFile(out, ZEPPELIN_PYTHON);
logger.info("File {} created", scriptPath);
}
public String getScriptPath() {
return scriptPath;
}
private void copyFile(File out, String sourceFile) {
ClassLoader classLoader = getClass().getClassLoader();
try {
FileOutputStream outStream = new FileOutputStream(out);
IOUtils.copy(
classLoader.getResourceAsStream(sourceFile),
outStream);
outStream.close();
} catch (IOException e) {
throw new InterpreterException(e);
}
}
private void createGatewayServerAndStartScript() throws UnknownHostException {
createPythonScript();
if (System.getenv("ZEPPELIN_HOME") != null) {
py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
pythonLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PYTHON_LIBS;
} else {
Path workingPath = Paths.get("..").toAbsolutePath();
py4jLibPath = workingPath + File.separator + ZEPPELIN_PY4JPATH;
pythonLibPath = workingPath + File.separator + ZEPPELIN_PYTHON_LIBS;
}
port = findRandomOpenPortOnAllLocalInterfaces();
gatewayServer = new GatewayServer(this,
port,
GatewayServer.DEFAULT_PYTHON_PORT,
InetAddress.getByName("0.0.0.0"),
InetAddress.getByName("0.0.0.0"),
GatewayServer.DEFAULT_CONNECT_TIMEOUT,
GatewayServer.DEFAULT_READ_TIMEOUT,
(List) null);
gatewayServer.start();
// Run python shell
String pythonCmd = getPythonCommand();
CommandLine cmd = CommandLine.parse(pythonCmd);
if (!pythonCmd.endsWith(".py")) {
// PythonDockerInterpreter set pythoncmd with script
cmd.addArgument(getScriptPath(), false);
}
cmd.addArgument(Integer.toString(port), false);
cmd.addArgument(getLocalIp(), false);
executor = new DefaultExecutor();
outputStream = new InterpreterOutputStream(logger);
PipedOutputStream ps = new PipedOutputStream();
in = null;
try {
in = new PipedInputStream(ps);
} catch (IOException e1) {
throw new InterpreterException(e1);
}
ins = new BufferedWriter(new OutputStreamWriter(ps));
input = new ByteArrayOutputStream();
PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream, outputStream, in);
executor.setStreamHandler(streamHandler);
executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
try {
Map env = EnvironmentUtils.getProcEnvironment();
if (!env.containsKey("PYTHONPATH")) {
env.put("PYTHONPATH", py4jLibPath + File.pathSeparator + pythonLibPath);
} else {
env.put("PYTHONPATH", env.get("PYTHONPATH") + File.pathSeparator +
py4jLibPath + File.pathSeparator + pythonLibPath);
}
logger.info("cmd = {}", cmd.toString());
executor.execute(cmd, env, this);
pythonscriptRunning = true;
} catch (IOException e) {
throw new InterpreterException(e);
}
try {
input.write("import sys, getopt\n".getBytes());
ins.flush();
} catch (IOException e) {
throw new InterpreterException(e);
}
}
@Override
@ -72,66 +222,99 @@ public class PythonInterpreter extends Interpreter {
// Add matplotlib display hook
InterpreterGroup intpGroup = getInterpreterGroup();
if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
registerHook(HookType.POST_EXEC_DEV, "\nz._displayhook()");
registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
}
// Add zeppelin-bundled libs to PYTHONPATH
setPythonPath("../interpreter/lib/python:$PYTHONPATH");
LOG.info("Starting Python interpreter ---->");
LOG.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
maxResult = Integer.valueOf(getProperty(MAX_RESULT));
process = getPythonProcess();
// Add matplotlib display hook
try {
process.open();
} catch (IOException e) {
LOG.error("Can't start the python process", e);
}
try {
LOG.info("python PID : " + process.getPid());
} catch (Exception e) {
LOG.warn("Can't find python pid process", e);
}
try {
LOG.info("Bootstrap interpreter with " + BOOTSTRAP_PY);
bootStrapInterpreter(BOOTSTRAP_PY);
} catch (IOException e) {
LOG.error("Can't execute " + BOOTSTRAP_PY + " to initiate python process", e);
}
py4JisInstalled = isPy4jInstalled();
if (py4JisInstalled) {
port = findRandomOpenPortOnAllLocalInterfaces();
LOG.info("Py4j gateway port : " + port);
try {
gatewayServer = new GatewayServer(this, port);
gatewayServer.start();
LOG.info("Bootstrap inputs with " + BOOTSTRAP_INPUT_PY);
bootStrapInterpreter(BOOTSTRAP_INPUT_PY);
} catch (IOException e) {
LOG.error("Can't execute " + BOOTSTRAP_INPUT_PY + " to " +
"initialize Zeppelin inputs in python process", e);
}
createGatewayServerAndStartScript();
} catch (UnknownHostException e) {
throw new InterpreterException(e);
}
}
@Override
public void close() {
LOG.info("closing Python interpreter <----");
pythonscriptRunning = false;
pythonScriptInitialized = false;
try {
if (process != null) {
process.close();
process = null;
}
if (gatewayServer != null) {
gatewayServer.shutdown();
}
ins.flush();
ins.close();
input.flush();
input.close();
} catch (IOException e) {
LOG.error("Can't close the interpreter", e);
e.printStackTrace();
}
executor.getWatchdog().destroyProcess();
new File(scriptPath).delete();
gatewayServer.shutdown();
// wait until getStatements stop
synchronized (statementSetNotifier) {
try {
statementSetNotifier.wait(1500);
} catch (InterruptedException e) {
}
statementSetNotifier.notify();
}
}
PythonInterpretRequest pythonInterpretRequest = null;
/**
* Result class of python interpreter
*/
public class PythonInterpretRequest {
public String statements;
public PythonInterpretRequest(String statements) {
this.statements = statements;
}
public String statements() {
return statements;
}
}
public PythonInterpretRequest getStatements() {
synchronized (statementSetNotifier) {
while (pythonInterpretRequest == null && pythonscriptRunning && pythonScriptInitialized) {
try {
statementSetNotifier.wait(1000);
} catch (InterruptedException e) {
}
}
PythonInterpretRequest req = pythonInterpretRequest;
pythonInterpretRequest = null;
return req;
}
}
String statementOutput = null;
boolean statementError = false;
Integer statementFinishedNotifier = new Integer(0);
public void setStatementsFinished(String out, boolean error) {
synchronized (statementFinishedNotifier) {
statementOutput = out;
statementError = error;
statementFinishedNotifier.notify();
}
}
boolean pythonScriptInitialized = false;
Integer pythonScriptInitializeNotifier = new Integer(0);
public void onPythonScriptInitialized(long pid) {
pythonPid = pid;
synchronized (pythonScriptInitializeNotifier) {
pythonScriptInitialized = true;
pythonScriptInitializeNotifier.notifyAll();
}
}
public void appendOutput(String message) throws IOException {
outputStream.getInterpreterOutput().write(message);
}
@Override
@ -139,44 +322,99 @@ public class PythonInterpreter extends Interpreter {
if (cmd == null || cmd.isEmpty()) {
return new InterpreterResult(Code.SUCCESS, "");
}
this.context = contextInterpreter;
String output = sendCommandToPython(cmd);
InterpreterResult result;
if (pythonErrorIn(output)) {
result = new InterpreterResult(Code.ERROR, output.replaceAll("\\.\\.\\.", ""));
} else {
result = new InterpreterResult(Code.SUCCESS, output);
if (!pythonscriptRunning) {
return new InterpreterResult(Code.ERROR, "python process not running"
+ outputStream.toString());
}
return result;
}
/**
* Checks if there is a syntax error or an exception
*
* @param output Python interpreter output
* @return true if syntax error or exception has happened
*/
private boolean pythonErrorIn(String output) {
boolean isError = false;
String[] outputMultiline = output.split("\n");
Matcher errorMatcher;
for (String row : outputMultiline) {
errorMatcher = errorInLastLine.matcher(row);
if (errorMatcher.find() == true) {
isError = true;
break;
outputStream.setInterpreterOutput(context.out);
synchronized (pythonScriptInitializeNotifier) {
long startTime = System.currentTimeMillis();
while (pythonScriptInitialized == false
&& pythonscriptRunning
&& System.currentTimeMillis() - startTime < MAX_TIMEOUT_SEC * 1000) {
try {
pythonScriptInitializeNotifier.wait(1000);
} catch (InterruptedException e) {
}
}
}
return isError;
List<InterpreterResultMessage> errorMessage;
try {
context.out.flush();
errorMessage = context.out.toInterpreterResultMessage();
} catch (IOException e) {
throw new InterpreterException(e);
}
if (pythonscriptRunning == false) {
// python script failed to initialize and terminated
errorMessage.add(new InterpreterResultMessage(
InterpreterResult.Type.TEXT, "failed to start python"));
return new InterpreterResult(Code.ERROR, errorMessage);
}
if (pythonScriptInitialized == false) {
// timeout. didn't get initialized message
errorMessage.add(new InterpreterResultMessage(
InterpreterResult.Type.TEXT, "python is not responding"));
return new InterpreterResult(Code.ERROR, errorMessage);
}
pythonInterpretRequest = new PythonInterpretRequest(cmd);
statementOutput = null;
synchronized (statementSetNotifier) {
statementSetNotifier.notify();
}
synchronized (statementFinishedNotifier) {
while (statementOutput == null) {
try {
statementFinishedNotifier.wait(1000);
} catch (InterruptedException e) {
}
}
}
if (statementError) {
return new InterpreterResult(Code.ERROR, statementOutput);
} else {
try {
context.out.flush();
} catch (IOException e) {
throw new InterpreterException(e);
}
return new InterpreterResult(Code.SUCCESS);
}
}
public InterpreterContext getCurrentInterpreterContext() {
return context;
}
public void interrupt() throws IOException {
if (pythonPid > -1) {
logger.info("Sending SIGINT signal to PID : " + pythonPid);
Runtime.getRuntime().exec("kill -SIGINT " + pythonPid);
} else {
logger.warn("Non UNIX/Linux system, close the interpreter");
close();
}
}
@Override
public void cancel(InterpreterContext context) {
try {
process.interrupt();
interrupt();
} catch (IOException e) {
LOG.error("Can't interrupt the python interpreter", e);
e.printStackTrace();
}
}
@ -201,28 +439,26 @@ public class PythonInterpreter extends Interpreter {
return null;
}
public void setPythonPath(String pythonPath) {
this.pythonPath = pythonPath;
}
public PythonProcess getPythonProcess() {
if (process == null) {
String binPath = getProperty(ZEPPELIN_PYTHON);
if (pythonCommand != null) {
binPath = pythonCommand;
}
return new PythonProcess(binPath, pythonPath);
} else {
return process;
}
}
public void setPythonCommand(String cmd) {
logger.info("Set Python Command : {}", cmd);
pythonCommand = cmd;
}
public String getPythonCommand() {
return pythonCommand;
private String getPythonCommand() {
if (pythonCommand == null) {
return getPythonBindPath();
} else {
return pythonCommand;
}
}
public String getPythonBindPath() {
String path = getProperty("zeppelin.python");
if (path == null) {
return DEFAULT_ZEPPELIN_PYTHON;
} else {
return path;
}
}
private Job getRunningJob(String paragraphId) {
@ -237,24 +473,6 @@ public class PythonInterpreter extends Interpreter {
return foundJob;
}
/**
* Sends given text to Python interpreter, blocks and returns the output
* @param cmd Python expression text
* @return output
*/
String sendCommandToPython(String cmd) {
String output = "";
LOG.debug("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 200) + "..." : cmd));
try {
output = process.sendAndGetResult(cmd);
} catch (IOException e) {
LOG.error("Error when sending commands to python process", e);
}
LOG.debug("Got : \n" + output);
return output;
}
void bootStrapInterpreter(String file) throws IOException {
BufferedReader bootstrapReader = new BufferedReader(
new InputStreamReader(
@ -265,24 +483,22 @@ public class PythonInterpreter extends Interpreter {
while ((line = bootstrapReader.readLine()) != null) {
bootstrapCode += line + "\n";
}
if (py4JisInstalled && port != null && port != -1) {
bootstrapCode = bootstrapCode.replaceAll("\\%PORT\\%", port.toString());
}
LOG.info("Bootstrap python interpreter with code from \n " + file);
sendCommandToPython(bootstrapCode);
interpret(bootstrapCode, context);
}
public GUI getGui() {
return context.getGui();
}
public Integer getPy4jPort() {
return port;
}
public Boolean isPy4jInstalled() {
String output = sendCommandToPython("\n\nimport py4j\n");
return !output.contains("ImportError");
String getLocalIp() {
try {
return Inet4Address.getLocalHost().getHostAddress();
} catch (UnknownHostException e) {
logger.error("can't get local IP", e);
}
// fall back to loopback addreess
return "127.0.0.1";
}
private int findRandomOpenPortOnAllLocalInterfaces() {
@ -299,4 +515,16 @@ public class PythonInterpreter extends Interpreter {
public int getMaxResult() {
return maxResult;
}
@Override
public void onProcessComplete(int exitValue) {
pythonscriptRunning = false;
logger.info("python process terminated. exit code " + exitValue);
}
@Override
public void onProcessFailed(ExecuteException e) {
pythonscriptRunning = false;
logger.error("python process failed", e);
}
}

17
python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java
View file

@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory;
public class PythonInterpreterPandasSql extends Interpreter {
private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreterPandasSql.class);
private String SQL_BOOTSTRAP_FILE_PY = "/bootstrap_sql.py";
private String SQL_BOOTSTRAP_FILE_PY = "/python/bootstrap_sql.py";
public PythonInterpreterPandasSql(Properties property) {
super(property);
@ -64,25 +64,17 @@ public class PythonInterpreterPandasSql extends Interpreter {
@Override
public void open() {
LOG.info("Open Python SQL interpreter instance: {}", this.toString());
try {
LOG.info("Bootstrap {} interpreter with {}", this.toString(), SQL_BOOTSTRAP_FILE_PY);
PythonInterpreter python = getPythonInterpreter();
python.bootStrapInterpreter(SQL_BOOTSTRAP_FILE_PY);
} catch (IOException e) {
LOG.error("Can't execute " + SQL_BOOTSTRAP_FILE_PY + " to import SQL dependencies", e);
}
}
/**
* Checks if Python dependencies pandas and pandasql are installed
* @return True if they are
*/
boolean isPandasAndPandasqlInstalled() {
PythonInterpreter python = getPythonInterpreter();
String output = python.sendCommandToPython("\n\nimport pandas\nimport pandasql\n");
return !output.contains("ImportError");
}
@Override
public void close() {
LOG.info("Close Python SQL interpreter instance: {}", this.toString());
@ -94,7 +86,8 @@ public class PythonInterpreterPandasSql extends Interpreter {
public InterpreterResult interpret(String st, InterpreterContext context) {
LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
Interpreter python = getPythonInterpreter();
return python.interpret("z.show(pysqldf('" + st + "'))", context);
return python.interpret("z.show(pysqldf('" + st + "'))\nz._displayhook()", context);
}
@Override

138
python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
View file

@ -1,138 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.OutputStream;
import java.lang.reflect.Field;
/**
* Object encapsulated interactive
* Python process (REPL) used by python interpreter
*/
public class PythonProcess {
private static final Logger logger = LoggerFactory.getLogger(PythonProcess.class);
private static final String STATEMENT_END = "*!?flush reader!?*";
InputStream stdout;
OutputStream stdin;
PrintWriter writer;
BufferedReader reader;
Process process;
private String binPath;
private String pythonPath;
private long pid;
public PythonProcess(String binPath, String pythonPath) {
this.binPath = binPath;
this.pythonPath = pythonPath;
}
public void open() throws IOException {
ProcessBuilder builder;
boolean hasParams = binPath.split(" ").length > 1;
if (System.getProperty("os.name").toLowerCase().contains("windows")) {
if (hasParams) {
builder = new ProcessBuilder(binPath.split(" "));
} else {
builder = new ProcessBuilder(binPath, "-iu");
}
} else {
String cmd;
if (hasParams) {
cmd = binPath;
} else {
cmd = binPath + " -iu";
}
builder = new ProcessBuilder("bash", "-c", cmd);
if (pythonPath != null) {
builder.environment().put("PYTHONPATH", pythonPath);
}
}
builder.redirectErrorStream(true);
process = builder.start();
stdout = process.getInputStream();
stdin = process.getOutputStream();
writer = new PrintWriter(stdin, true);
reader = new BufferedReader(new InputStreamReader(stdout));
try {
pid = findPid();
} catch (Exception e) {
logger.warn("Can't find python pid process", e);
pid = -1;
}
}
public void close() throws IOException {
process.destroy();
reader.close();
writer.close();
stdin.close();
stdout.close();
}
public void interrupt() throws IOException {
if (pid > -1) {
logger.info("Sending SIGINT signal to PID : " + pid);
Runtime.getRuntime().exec("kill -SIGINT " + pid);
} else {
logger.warn("Non UNIX/Linux system, close the interpreter");
close();
}
}
public String sendAndGetResult(String cmd) throws IOException {
writer.println(cmd);
writer.println();
writer.println("\"" + STATEMENT_END + "\"");
StringBuilder output = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null &&
!line.contains(STATEMENT_END)) {
logger.debug("Read line from python shell : " + line);
output.append(line + "\n");
}
return output.toString();
}
private long findPid() throws NoSuchFieldException, IllegalAccessException {
long pid = -1;
if (process.getClass().getName().equals("java.lang.UNIXProcess")) {
Field f = process.getClass().getDeclaredField("pid");
f.setAccessible(true);
pid = f.getLong(process);
f.setAccessible(false);
}
return pid;
}
public long getPid() {
return pid;
}
}

14
python/src/main/resources/__init__.py
View file

@ -1,14 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

234
python/src/main/resources/bootstrap.py
View file

@ -1,234 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# PYTHON 2 / 3 compatibility :
# bootstrap.py must be runnable with Python 2 or 3
import os
import sys
import signal
import base64
import warnings
from io import BytesIO
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
def intHandler(signum, frame): # Set the signal handler
print ("Paragraph interrupted")
raise KeyboardInterrupt()
signal.signal(signal.SIGINT, intHandler)
# set prompt as empty string so that java side don't need to remove the prompt.
sys.ps1=""
def help():
print("""%html
<h2>Python Interpreter help</h2>
<h3>Python 2 & 3 compatibility</h3>
<p>The interpreter is compatible with Python 2 & 3.<br/>
To change Python version,
change in the interpreter configuration the python to the
desired version (example : python=/usr/bin/python3)</p>
<h3>Python modules</h3>
<p>The interpreter can use all modules already installed
(with pip, easy_install, etc)</p>
<h3>Forms</h3>
You must install py4j in order to use
the form feature (pip install py4j)
<h4>Input form</h4>
<pre>print (z.input("f1","defaultValue"))</pre>
<h4>Selection form</h4>
<pre>print(z.select("f2", [("o1","1"), ("o2","2")],2))</pre>
<h4>Checkbox form</h4>
<pre> print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))</pre>')
<h3>Matplotlib graph</h3>
<div>The interpreter can display matplotlib graph with
the function z.show()</div>
<div> You need to already have matplotlib module installed
to use this functionality !</div><br/>
<pre>import matplotlib.pyplot as plt
plt.figure()
(.. ..)
z.show(plt)
plt.close()
</pre>
<div><br/> z.show function can take optional parameters
to adapt graph dimensions (width and height) and format (png or svg)</div>
<div><b>example </b>:
<pre>z.show(plt,width='50px
z.show(plt,height='150px', fmt='svg') </pre></div>
<h3>Pandas DataFrame</h3>
<div> You need to have Pandas module installed
to use this functionality (pip install pandas) !</div><br/>
<div>The interpreter can visualize Pandas DataFrame
with the function z.show()
<pre>
import pandas as pd
df = pd.read_csv("bank.csv", sep=";")
z.show(df)
</pre></div>
<h3>SQL over Pandas DataFrame</h3>
<div> You need to have Pandas&Pandasql modules installed
to use this functionality (pip install pandas pandasql) !</div><br/>
<div>Python interpreter group includes %sql interpreter that can query
Pandas DataFrames using SQL and visualize results using Zeppelin Table Display System
<pre>
%python
import pandas as pd
df = pd.read_csv("bank.csv", sep=";")
</pre>
<br />
<pre>
%python.sql
%sql
SELECT * from df LIMIT 5
</pre>
</div>
""")
class PyZeppelinContext(object):
""" If py4j is detected, these class will be override
with the implementation in bootstrap_input.py
"""
errorMsg = "You must install py4j Python module " \
"(pip install py4j) to use Zeppelin dynamic forms features"
def __init__(self):
self.max_result = 1000
self._displayhook = lambda *args: None
self._setup_matplotlib()
def input(self, name, defaultValue=""):
print(self.errorMsg)
def select(self, name, options, defaultValue=""):
print(self.errorMsg)
def checkbox(self, name, options, defaultChecked=[]):
print(self.errorMsg)
def show(self, p, **kwargs):
if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
self.show_matplotlib(p, **kwargs)
elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
# `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
# and so a dependency on pandas
self.show_dataframe(p, **kwargs)
elif hasattr(p, '__call__'):
p() #error reporting
def show_dataframe(self, df, show_index=False, **kwargs):
"""Pretty prints DF using Table Display System
"""
limit = len(df) > self.max_result
header_buf = StringIO("")
if show_index:
idx_name = str(df.index.name) if df.index.name is not None else ""
header_buf.write(idx_name + "\t")
header_buf.write(str(df.columns[0]))
for col in df.columns[1:]:
header_buf.write("\t")
header_buf.write(str(col))
header_buf.write("\n")
body_buf = StringIO("")
rows = df.head(self.max_result).values if limit else df.values
index = df.index.values
for idx, row in zip(index, rows):
if show_index:
body_buf.write("%html <strong>{}</strong>".format(idx))
body_buf.write("\t")
body_buf.write(str(row[0]))
for cell in row[1:]:
body_buf.write("\t")
body_buf.write(str(cell))
body_buf.write("\n")
body_buf.seek(0); header_buf.seek(0)
#TODO(bzz): fix it, so it shows red notice, as in Spark
print("%table " + header_buf.read() + body_buf.read()) # +
# ("\n<font color=red>Results are limited by {}.</font>" \
# .format(self.max_result) if limit else "")
#)
body_buf.close(); header_buf.close()
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
**kwargs):
"""Matplotlib show function
"""
if fmt == "png":
img = BytesIO()
p.savefig(img, format=fmt)
img_str = b"data:image/png;base64,"
img_str += base64.b64encode(img.getvalue().strip())
img_tag = "<img src={img} style='width={width};height:{height}'>"
# Decoding is necessary for Python 3 compability
img_str = img_str.decode("ascii")
img_str = img_tag.format(img=img_str, width=width, height=height)
elif fmt == "svg":
img = StringIO()
p.savefig(img, format=fmt)
img_str = img.getvalue()
else:
raise ValueError("fmt must be 'png' or 'svg'")
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
print(html.format(width=width, height=height, img=img_str))
img.close()
def configure_mpl(self, **kwargs):
import mpl_config
mpl_config.configure(**kwargs)
def _setup_matplotlib(self):
# If we don't have matplotlib installed don't bother continuing
try:
import matplotlib
except ImportError:
return
# Make sure custom backends are available in the PYTHONPATH
rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
if mpl_path not in sys.path:
sys.path.append(mpl_path)
# Finally check if backend exists, and if so configure as appropriate
try:
matplotlib.use('module://backend_zinline')
import backend_zinline
# Everything looks good so make config assuming that we are using
# an inline backend
self._displayhook = backend_zinline.displayhook
self.configure_mpl(width=600, height=400, dpi=72,
fontsize=10, interactive=True, format='png')
except ImportError:
# Fall back to Agg if no custom backend installed
matplotlib.use('Agg')
warnings.warn("Unable to load inline matplotlib backend, "
"falling back to Agg")
z = PyZeppelinContext()

58
python/src/main/resources/bootstrap_input.py
View file

@ -1,58 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from py4j.java_gateway import JavaGateway
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
client = GatewayClient(port=%PORT%)
gateway = JavaGateway(client)
java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
class Py4jZeppelinContext(PyZeppelinContext):
"""A context impl that uses Py4j to communicate to JVM
"""
def __init__(self, z):
PyZeppelinContext.__init__(self)
self.z = z
self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
self.javaList = gateway.jvm.java.util.ArrayList
self.max_result = self.z.getMaxResult()
def input(self, name, defaultValue=""):
return self.z.getGui().input(name, defaultValue)
def select(self, name, options, defaultValue=""):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
for tuple in options:
javaOptions[i] = self.paramOption(tuple[0], tuple[1])
i += 1
return self.z.getGui().select(name, defaultValue, javaOptions)
def checkbox(self, name, options, defaultChecked=[]):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
for tuple in options:
javaOptions[i] = self.paramOption(tuple[0], tuple[1])
i += 1
javaDefaultCheck = self.javaList()
for check in defaultChecked:
javaDefaultCheck.append(check)
return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
z = Py4jZeppelinContext(gateway.entry_point)

2
python/src/main/resources/interpreter-setting.json
View file

@ -39,7 +39,7 @@
"className": "org.apache.zeppelin.python.PythonCondaInterpreter",
"properties": {
},
"editor":{
"editor": {
"language": "sh",
"editOnDblClick": false
}

9
python/src/main/resources/bootstrap_sql.py → python/src/main/resources/python/bootstrap_sql.py
View file

@ -21,8 +21,9 @@
from __future__ import print_function
try:
from pandasql import sqldf
pysqldf = lambda q: sqldf(q, globals())
from pandasql import sqldf
pysqldf = lambda q: sqldf(q, globals())
except ImportError:
pysqldf = lambda q: print("Can not run SQL over Pandas DataFrame" +
"Make sure 'pandas' and 'pandasql' libraries are installed")
pysqldf = lambda q: print("Can not run SQL over Pandas DataFrame" +
"Make sure 'pandas' and 'pandasql' libraries are installed")

293
python/src/main/resources/python/zeppelin_python.py Normal file
View file

@ -0,0 +1,293 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os, sys, getopt, traceback, json, re
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
from py4j.protocol import Py4JJavaError, Py4JNetworkError
import warnings
import ast
import traceback
import warnings
import signal
from io import BytesIO
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
# for back compatibility
class Logger(object):
def __init__(self):
pass
def write(self, message):
intp.appendOutput(message)
def reset(self):
pass
def flush(self):
pass
class PyZeppelinContext(object):
""" A context impl that uses Py4j to communicate to JVM
"""
def __init__(self, z):
self.z = z
self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
self.javaList = gateway.jvm.java.util.ArrayList
self.max_result = 1000
self._displayhook = lambda *args: None
self._setup_matplotlib()
def getInterpreterContext(self):
return self.z.getCurrentInterpreterContext()
def input(self, name, defaultValue=""):
return self.z.getGui().input(name, defaultValue)
def select(self, name, options, defaultValue=""):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
for tuple in options:
javaOptions[i] = self.paramOption(tuple[0], tuple[1])
i += 1
return self.z.getGui().select(name, defaultValue, javaOptions)
def checkbox(self, name, options, defaultChecked=[]):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
for tuple in options:
javaOptions[i] = self.paramOption(tuple[0], tuple[1])
i += 1
javaDefaultCheck = self.javaList()
for check in defaultChecked:
javaDefaultCheck.append(check)
return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
def show(self, p, **kwargs):
if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
self.show_matplotlib(p, **kwargs)
elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
# `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
# and so a dependency on pandas
self.show_dataframe(p, **kwargs)
elif hasattr(p, '__call__'):
p() #error reporting
def show_dataframe(self, df, show_index=False, **kwargs):
"""Pretty prints DF using Table Display System
"""
limit = len(df) > self.max_result
header_buf = StringIO("")
if show_index:
idx_name = str(df.index.name) if df.index.name is not None else ""
header_buf.write(idx_name + "\t")
header_buf.write(str(df.columns[0]))
for col in df.columns[1:]:
header_buf.write("\t")
header_buf.write(str(col))
header_buf.write("\n")
body_buf = StringIO("")
rows = df.head(self.max_result).values if limit else df.values
index = df.index.values
for idx, row in zip(index, rows):
if show_index:
body_buf.write("%html <strong>{}</strong>".format(idx))
body_buf.write("\t")
body_buf.write(str(row[0]))
for cell in row[1:]:
body_buf.write("\t")
body_buf.write(str(cell))
body_buf.write("\n")
body_buf.seek(0); header_buf.seek(0)
#TODO(bzz): fix it, so it shows red notice, as in Spark
print("%table " + header_buf.read() + body_buf.read()) # +
# ("\n<font color=red>Results are limited by {}.</font>" \
# .format(self.max_result) if limit else "")
#)
body_buf.close(); header_buf.close()
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
**kwargs):
"""Matplotlib show function
"""
if fmt == "png":
img = BytesIO()
p.savefig(img, format=fmt)
img_str = b"data:image/png;base64,"
img_str += base64.b64encode(img.getvalue().strip())
img_tag = "<img src={img} style='width={width};height:{height}'>"
# Decoding is necessary for Python 3 compability
img_str = img_str.decode("ascii")
img_str = img_tag.format(img=img_str, width=width, height=height)
elif fmt == "svg":
img = StringIO()
p.savefig(img, format=fmt)
img_str = img.getvalue()
else:
raise ValueError("fmt must be 'png' or 'svg'")
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
print(html.format(width=width, height=height, img=img_str))
img.close()
def configure_mpl(self, **kwargs):
import mpl_config
mpl_config.configure(**kwargs)
def _setup_matplotlib(self):
# If we don't have matplotlib installed don't bother continuing
try:
import matplotlib
except ImportError:
return
# Make sure custom backends are available in the PYTHONPATH
rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
if mpl_path not in sys.path:
sys.path.append(mpl_path)
# Finally check if backend exists, and if so configure as appropriate
try:
matplotlib.use('module://backend_zinline')
import backend_zinline
# Everything looks good so make config assuming that we are using
# an inline backend
self._displayhook = backend_zinline.displayhook
self.configure_mpl(width=600, height=400, dpi=72,
fontsize=10, interactive=True, format='png')
except ImportError:
# Fall back to Agg if no custom backend installed
matplotlib.use('Agg')
warnings.warn("Unable to load inline matplotlib backend, "
"falling back to Agg")
def handler_stop_signals(sig, frame):
sys.exit("Got signal : " + str(sig))
signal.signal(signal.SIGINT, handler_stop_signals)
host = "127.0.0.1"
if len(sys.argv) >= 3:
host = sys.argv[2]
client = GatewayClient(address=host, port=int(sys.argv[1]))
#gateway = JavaGateway(client, auto_convert = True)
gateway = JavaGateway(client)
intp = gateway.entry_point
intp.onPythonScriptInitialized(os.getpid())
java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
z = PyZeppelinContext(intp)
z._setup_matplotlib()
output = Logger()
sys.stdout = output
#sys.stderr = output
while True :
req = intp.getStatements()
if req == None:
break
try:
stmts = req.statements().split("\n")
final_code = []
# Get post-execute hooks
try:
global_hook = intp.getHook('post_exec_dev')
except:
global_hook = None
try:
user_hook = z.getHook('post_exec')
except:
user_hook = None
nhooks = 0
for hook in (global_hook, user_hook):
if hook:
nhooks += 1
for s in stmts:
if s == None:
continue
# skip comment
s_stripped = s.strip()
if len(s_stripped) == 0 or s_stripped.startswith("#"):
continue
final_code.append(s)
if final_code:
# use exec mode to compile the statements except the last statement,
# so that the last statement's evaluation will be printed to stdout
code = compile('\n'.join(final_code), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
to_run_hooks = []
if (nhooks > 0):
to_run_hooks = code.body[-nhooks:]
to_run_exec, to_run_single = (code.body[:-(nhooks + 1)],
[code.body[-(nhooks + 1)]])
try:
for node in to_run_exec:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec(code)
for node in to_run_single:
mod = ast.Interactive([node])
code = compile(mod, '<stdin>', 'single')
exec(code)
for node in to_run_hooks:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec(code)
except:
raise Exception(traceback.format_exc())
intp.setStatementsFinished("", False)
except Py4JJavaError:
excInnerError = traceback.format_exc() # format_tb() does not return the inner exception
innerErrorStart = excInnerError.find("Py4JJavaError:")
if innerErrorStart > -1:
excInnerError = excInnerError[innerErrorStart:]
intp.setStatementsFinished(excInnerError + str(sys.exc_info()), True)
except Py4JNetworkError:
# lost connection from gateway server. exit
sys.exit(1)
except:
intp.setStatementsFinished(traceback.format_exc(), True)
output.reset()

34
python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java
View file

@ -1,21 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
@ -132,4 +134,6 @@ public class PythonCondaInterpreterTest {
null,
new InterpreterOutput(null));
}
}

9
python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java
View file

@ -21,8 +21,11 @@ import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;
@ -46,8 +49,12 @@ public class PythonDockerInterpreterTest {
group.put("note", Arrays.asList(python, docker));
python.setInterpreterGroup(group);
docker.setInterpreterGroup(group);
doReturn(true).when(docker).pull(any(InterpreterOutput.class), anyString());
doReturn(python).when(docker).getPythonInterpreter();
doReturn("/scriptpath/zeppelin_python.py").when(python).getScriptPath();
docker.open();
}
@Test
@ -57,7 +64,7 @@ public class PythonDockerInterpreterTest {
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(docker, times(1)).pull(any(InterpreterOutput.class), anyString());
verify(python).setPythonCommand("docker run -i --rm env python -iu");
verify(python).setPythonCommand(Mockito.matches("docker run -i --rm -v.*"));
}
@Test

146
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java
View file

@ -1,25 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import java.util.*;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
@ -29,34 +26,27 @@ import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import static org.junit.Assert.*;
/**
* In order for this test to work, test env must have installed:
* <ol>
* - <li>Python</li>
* - <li>Matplotlib</li>
* <ol>
*
* Your PYTHONPATH should also contain the directory of the Matplotlib
* backend files. Usually these can be found in $ZEPPELIN_HOME/interpreter/lib/python.
*
* To run manually on such environment, use:
* <code>
* mvn -Dpython.test.exclude='' test -pl python -am
* </code>
*/
public class PythonInterpreterMatplotlibTest {
public class PythonInterpreterMatplotlibTest implements InterpreterOutputListener {
private InterpreterGroup intpGroup;
private PythonInterpreter python;
private InterpreterContext context;
InterpreterOutput out;
@Before
public void setUp() throws Exception {
@ -68,16 +58,27 @@ public class PythonInterpreterMatplotlibTest {
python = new PythonInterpreter(p);
python.setInterpreterGroup(intpGroup);
python.open();
List<Interpreter> interpreters = new LinkedList<>();
interpreters.add(python);
intpGroup.put("note", interpreters);
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
out = new InterpreterOutput(this);
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
out);
python.open();
}
@After
public void afterTest() throws IOException {
python.close();
}
@Test
@ -85,14 +86,14 @@ public class PythonInterpreterMatplotlibTest {
// matplotlib
InterpreterResult ret = python.interpret("import matplotlib", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
// inline backend
ret = python.interpret("import backend_zinline", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
}
@Test
public void showPlot() {
public void showPlot() throws IOException {
// Simple plot test
InterpreterResult ret;
ret = python.interpret("import matplotlib.pyplot as plt", context);
@ -100,15 +101,16 @@ public class PythonInterpreterMatplotlibTest {
ret = python.interpret("plt.plot([1, 2, 3])", context);
ret = python.interpret("plt.show()", context);
assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(ret.message().get(0).getData(), Type.HTML, ret.message().get(0).getType());
assertTrue(ret.message().get(0).getData().contains("data:image/png;base64"));
assertTrue(ret.message().get(0).getData().contains("<div>"));
assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Type.TEXT, out.getOutputAt(0).getType());
assertEquals(new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Type.HTML, out.getOutputAt(1).getType());
assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("data:image/png;base64"));
assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("<div>"));
}
@Test
// Test for when configuration is set to auto-close figures after show().
public void testClose() {
public void testClose() throws IOException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;
@ -116,25 +118,33 @@ public class PythonInterpreterMatplotlibTest {
ret = python.interpret("z.configure_mpl(interactive=False)", context);
ret = python.interpret("plt.plot([1, 2, 3])", context);
ret1 = python.interpret("plt.show()", context);
// Second call to show() should print nothing, and Type should be TEXT.
// This is because when close=True, there should be no living instances
// of FigureManager, causing show() to return before setting the output
// type to HTML.
ret = python.interpret("plt.show()", context);
assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(0, ret.message().size());
// Now test that new plot is drawn. It should be identical to the
// previous one.
ret = python.interpret("plt.plot([1, 2, 3])", context);
String msg1 = new String(out.getOutputAt(0).toByteArray());
InterpreterResult.Type type1 = out.getOutputAt(0).getType();
ret2 = python.interpret("plt.show()", context);
assertEquals(ret1.message().get(0).getType(), ret2.message().get(0).getType());
assertEquals(ret1.message().get(0).getData(), ret2.message().get(0).getData());
String msg2 = new String(out.getOutputAt(0).toByteArray());
InterpreterResult.Type type2 = out.getOutputAt(0).getType();
assertEquals(msg1, msg2);
assertEquals(type1, type2);
}
@Test
// Test for when configuration is set to not auto-close figures after show().
public void testNoClose() {
public void testNoClose() throws IOException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;
@ -142,19 +152,39 @@ public class PythonInterpreterMatplotlibTest {
ret = python.interpret("z.configure_mpl(interactive=False, close=False)", context);
ret = python.interpret("plt.plot([1, 2, 3])", context);
ret1 = python.interpret("plt.show()", context);
// Second call to show() should print nothing, and Type should be HTML.
// This is because when close=False, there should be living instances
// of FigureManager, causing show() to set the output
// type to HTML even though the figure is inactive.
ret = python.interpret("plt.show()", context);
assertEquals("", ret.message().get(0).getData());
String msg1 = new String(out.getOutputAt(0).toByteArray());
assertNotSame("", msg1);
// Now test that plot can be reshown if it is updated. It should be
// different from the previous one because it will plot the same line
// again but in a different color.
ret = python.interpret("plt.plot([1, 2, 3])", context);
msg1 = new String(out.getOutputAt(1).toByteArray());
ret2 = python.interpret("plt.show()", context);
assertNotSame(ret1.message().get(0).getData(), ret2.message().get(0).getData());
String msg2 = new String(out.getOutputAt(1).toByteArray());
assertNotSame(msg1, msg2);
}
@Override
public void onUpdateAll(InterpreterOutput out) {
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
}
}

94
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java
View file

@ -21,13 +21,16 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@ -35,7 +38,10 @@ import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -53,13 +59,14 @@ import org.junit.Test;
* mvn -Dpython.test.exclude='' test -pl python -am
* </code>
*/
public class PythonInterpreterPandasSqlTest {
public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener {
private InterpreterGroup intpGroup;
private PythonInterpreterPandasSql sql;
private PythonInterpreter python;
private InterpreterContext context;
InterpreterOutput out;
@Before
public void setUp() throws Exception {
@ -78,14 +85,27 @@ public class PythonInterpreterPandasSqlTest {
intpGroup.put("note", Arrays.asList(python, sql));
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));
out = new InterpreterOutput(this);
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
out);
// to make sure python is running.
InterpreterResult ret = python.interpret("\n", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
//important to be last step
sql.open();
//it depends on python interpreter presence in the same group
}
@After
public void afterTest() throws IOException {
sql.close();
}
@Test
@ -97,23 +117,15 @@ public class PythonInterpreterPandasSqlTest {
@Test
public void errorMessageIfDependenciesNotInstalled() {
InterpreterResult ret;
// given
ret = python.interpret(
"pysqldf = lambda q: print('Can not execute SQL as Python dependency is not installed')",
context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
// when
ret = sql.interpret("SELECT * from something", context);
// then
assertNotNull(ret);
assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().get(0).getData().contains("dependency is not installed"));
assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().get(0).getData().contains("no such table: something"));
}
@Test
public void sqlOverTestDataPrintsTable() {
public void sqlOverTestDataPrintsTable() throws IOException {
InterpreterResult ret;
// given
//String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34";
@ -121,36 +133,34 @@ public class PythonInterpreterPandasSqlTest {
ret = python.interpret("import numpy as np", context);
// DataFrame df2 \w test data
ret = python.interpret("df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), "+
"'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
"'name' : pd.Categorical(['moon','jobs','gates','park'])})", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
//when
ret = sql.interpret("select name, age from df2 where age < 40", context);
//then
assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(ret.message().get(0).getData(), Type.TABLE, ret.message().get(0).getType());
//assertEquals(expectedTable, ret.message()); //somehow it's same but not equal
assertTrue(ret.message().get(0).getData().indexOf("moon\t33") > 0);
assertTrue(ret.message().get(0).getData().indexOf("park\t34") > 0);
assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("moon\t33") > 0);
assertTrue(new String(out.getOutputAt(0).toByteArray()).indexOf("park\t34") > 0);
assertEquals(InterpreterResult.Code.SUCCESS, sql.interpret("select case when name==\"aa\" then name else name end from df2", context).code());
}
@Test
public void badSqlSyntaxFails() {
public void badSqlSyntaxFails() throws IOException {
//when
InterpreterResult ret = sql.interpret("select wrong syntax", context);
//then
assertNotNull("Interpreter returned 'null'", ret);
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);
assertTrue(out.toInterpreterResultMessage().size() == 0);
}
@Test
public void showDataFrame() {
public void showDataFrame() throws IOException {
InterpreterResult ret;
ret = python.interpret("import pandas as pd", context);
ret = python.interpret("import numpy as np", context);
@ -165,11 +175,25 @@ public class PythonInterpreterPandasSqlTest {
ret = python.interpret("z.show(df1, show_index=True)", context);
// then
assertEquals(ret.message().get(0).getData(), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(ret.message().get(0).getData(), Type.TABLE, ret.message().get(0).getType());
assertTrue(ret.message().get(0).getData().indexOf("index_name") == 0);
assertTrue(ret.message().get(0).getData().indexOf("13") > 0);
assertTrue(ret.message().get(0).getData().indexOf("nan") > 0);
assertTrue(ret.message().get(0).getData().indexOf("6.7") > 0);
assertEquals(new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code());
assertEquals(new String(out.getOutputAt(0).toByteArray()), Type.TABLE, out.getOutputAt(0).getType());
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("index_name"));
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("nan"));
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("6.7"));
}
}
@Override
public void onUpdateAll(InterpreterOutput out) {
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
}
}

278
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -1,19 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
@ -21,52 +21,39 @@ import static org.apache.zeppelin.python.PythonInterpreter.DEFAULT_ZEPPELIN_PYTH
import static org.apache.zeppelin.python.PythonInterpreter.MAX_RESULT;
import static org.apache.zeppelin.python.PythonInterpreter.ZEPPELIN_PYTHON;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Python interpreter unit test
*
* Important: ALL tests here DO NOT REQUIRE Python to be installed
* If Python dependency is required, please look at PythonInterpreterWithPythonInstalledTest
*/
public class PythonInterpreterTest {
private static final Logger LOG = LoggerFactory.getLogger(PythonProcess.class);
PythonInterpreter zeppelinPythonInterpreter = null;
public class PythonInterpreterTest implements InterpreterOutputListener {
PythonInterpreter pythonInterpreter = null;
PythonProcess mockPythonProcess;
String cmdHistory;
private InterpreterContext context;
InterpreterOutput out;
public static Properties getPythonTestProperties() {
Properties p = new Properties();
@ -79,18 +66,8 @@ public class PythonInterpreterTest {
public void beforeTest() throws IOException {
cmdHistory = "";
/*Mock python process*/
mockPythonProcess = mock(PythonProcess.class);
when(mockPythonProcess.getPid()).thenReturn(1L);
when(mockPythonProcess.sendAndGetResult(anyString())).thenAnswer(new Answer<String>() {
@Override public String answer(InvocationOnMock invocationOnMock) throws Throwable {
return answerFromPythonMock(invocationOnMock);
}
});
// python interpreter
pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties()));
zeppelinPythonInterpreter = new PythonInterpreter(getPythonTestProperties());
pythonInterpreter = new PythonInterpreter(getPythonTestProperties());
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
@ -98,186 +75,49 @@ public class PythonInterpreterTest {
group.get("note").add(pythonInterpreter);
pythonInterpreter.setInterpreterGroup(group);
when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess);
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError");
out = new InterpreterOutput(this);
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(group.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
out);
pythonInterpreter.open();
}
@After
public void afterTest() throws IOException {
pythonInterpreter.close();
zeppelinPythonInterpreter.close();
}
@Test
public void testOpenInterpreter() {
pythonInterpreter.open();
assertEquals(pythonInterpreter.getPythonProcess().getPid(), 1);
}
/**
* If Py4J is not installed, bootstrap_input.py
* is not sent to Python process and py4j JavaGateway is not running
*/
@Test
public void testPy4jIsNotInstalled() {
pythonInterpreter.open();
assertNull(pythonInterpreter.getPy4jPort());
assertTrue(cmdHistory.contains("def help()"));
assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
assertTrue(cmdHistory.contains("z = PyZeppelinContext"));
assertTrue(cmdHistory.contains("def show"));
assertFalse(cmdHistory.contains("GatewayClient"));
}
/**
* If Py4J installed, bootstrap_input.py
* is sent to interpreter and JavaGateway is running
*/
@Test
public void testPy4jInstalled() throws IOException, InterruptedException {
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
pythonInterpreter.open();
Integer py4jPort = pythonInterpreter.getPy4jPort();
assertNotNull(py4jPort);
assertTrue(cmdHistory.contains("def help()"));
assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
assertTrue(cmdHistory.contains("z = Py4jZeppelinContext"));
assertTrue(cmdHistory.contains("def show"));
assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")"));
assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input"));
assertTrue(serverIsListeningOn(py4jPort));
pythonInterpreter.close();
TimeUnit.MILLISECONDS.sleep(100);
assertFalse(serverIsListeningOn(py4jPort));
}
@Test
public void testClose() throws IOException, InterruptedException {
//given: py4j is installed
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
pythonInterpreter.open();
Integer py4jPort = pythonInterpreter.getPy4jPort();
assertNotNull(py4jPort);
//when
pythonInterpreter.close();
TimeUnit.MILLISECONDS.sleep(100);
//then
assertFalse(serverIsListeningOn(py4jPort));
verify(mockPythonProcess, times(1)).close();
}
@Test
public void testInterpret() {
pythonInterpreter.open();
cmdHistory = "";
InterpreterResult result = pythonInterpreter.interpret("print a", null);
public void testInterpret() throws InterruptedException, IOException {
InterpreterResult result = pythonInterpreter.interpret("print (\"hi\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals("%text print a", result.message().get(0).toString());
}
@Test
public void testInterpretInvalidSyntax() {
zeppelinPythonInterpreter.open();
InterpreterResult result = zeppelinPythonInterpreter.interpret("for x in range(0,3): print (\"hi\")\n\nz._displayhook()", null);
public void testInterpretInvalidSyntax() throws IOException {
InterpreterResult result = pythonInterpreter.interpret("for x in range(0,3): print (\"hi\")\n", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().get(0).toString().contains("hi\nhi\nhi"));
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
}
@Override
public void onUpdateAll(InterpreterOutput out) {
result = zeppelinPythonInterpreter.interpret("for x in range(0,3): print (\"hi\")\nz._displayhook()", null);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).toString().contains("SyntaxError: invalid syntax"));
}
/**
* Checks if given port is open on 'localhost'
* @param port
*/
private boolean serverIsListeningOn(Integer port) {
Socket s = new Socket();
boolean serverIsListening = false;
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
int retryCount = 0;
boolean connected = false;
while (connected = tryToConnect(s, port) && retryCount < 10) {
serverIsListening = connected;
tryToClose(s);
retryCount++;
s = new Socket();
}
return serverIsListening;
}
private boolean tryToConnect(Socket s, Integer port) {
boolean connected = false;
SocketAddress sa = new InetSocketAddress("localhost", port);
try {
s.connect(sa, 10000);
connected = true;
} catch (IOException e) {
//LOG.warn("Can't open connection to " + sa, e);
}
return connected;
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
}
private void tryToClose(Socket s) {
try {
s.close();
} catch (IOException e) {
LOG.error("Can't close connection to " + s.getInetAddress(), e);
}
}
private String answerFromPythonMock(InvocationOnMock invocationOnMock) {
Object[] inputs = invocationOnMock.getArguments();
String cmdToExecute = (String) inputs[0];
if (cmdToExecute != null) {
cmdHistory += cmdToExecute;
String[] lines = cmdToExecute.split("\\n");
String output = "";
for (int i = 0; i < lines.length; i++) {
output += lines[i];
}
return output;
} else {
return "";
}
}
@Test
public void checkMultiRowErrorFails() {
PythonInterpreter pythonInterpreter = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties()
);
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
group.put("note", new LinkedList<Interpreter>());
group.get("note").add(pythonInterpreter);
pythonInterpreter.setInterpreterGroup(group);
pythonInterpreter.open();
String codeRaiseException = "raise Exception(\"test exception\")";
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
assertNotNull("Interpreter result for raise exception is Null", ret);
System.err.println("ret = '" + ret + "'");
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);
assertNotNull("Interpreter result for text is Null", ret);
String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
ret = pythonInterpreter.interpret(codePrintText, null);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);
}
}

125
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
View file

@ -1,125 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Test;
import java.util.Arrays;
/**
* Python interpreter unit test that user real Python
*
* Important: ALL tests here REQUIRE Python to be installed
* They are excluded from default build, to run them manually do:
*
* <code>
* mvn "-Dtest=org.apache.zeppelin.python.PythonInterpreterWithPythonInstalledTest" test -pl python
* </code>
*
* or
* <code>
* mvn -Dpython.test.exclude='' test -pl python -am
* </code>
*/
public class PythonInterpreterWithPythonInstalledTest {
@Test
public void badPythonSyntaxFails() {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when
InterpreterResult ret = realPython.interpret("select wrong syntax", null);
//then
assertNotNull("Interpreter returned 'null'", ret);
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);
realPython.close();
}
@Test
public void goodPythonSyntaxRuns() {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when
InterpreterResult ret = realPython.interpret("help()", null);
//then
assertNotNull("Interpreter returned 'null'", ret);
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);
realPython.close();
}
@Test
public void testZeppelin1555() {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when
InterpreterResult ret1 = realPython.interpret("print(\"...\")", null);
//then
//System.out.println("\nInterpreter response: \n" + ret.message());
assertEquals(InterpreterResult.Code.SUCCESS, ret1.code());
assertEquals("...\n", ret1.message().get(0).getData());
InterpreterResult ret2 = realPython.interpret("for i in range(5):", null);
//then
//System.out.println("\nInterpreterResultterpreter response: \n" + ret2.message());
assertEquals(InterpreterResult.Code.ERROR, ret2.code());
assertEquals(" File \"<stdin>\", line 2\n" +
" \n" +
" ^\n" +
"IndentationError: expected an indented block\n", ret2.message().get(0).getData());
realPython.close();
}
}

37
r/pom.xml
View file

@ -190,43 +190,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>lib/**</exclude>
<exclude>**/r/lib/**</exclude>
<!--The following files are subject to the BSD-license or variants,
as shown in the file headers-->
<exclude>**/R/rzeppelin/R/globals.R</exclude>
<exclude>**/R/rzeppelin/R/common.R</exclude>
<exclude>**/R/rzeppelin/R/protocol.R</exclude>
<exclude>**/R/rzeppelin/R/rServer.R</exclude>
<exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude>
<exclude>**/R/rzeppelin/R/zzz.R</exclude>
<exclude>**/scala/Console.scala</exclude>
<exclude>**/zeppelin/rinterpreter/rscala/Package.scala</exclude>
<exclude>**/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
<!--End of files subject to BSD-license.-->
<exclude>**/.idea/</exclude>
<!--The following files are mechanical-->
<exclude>**/R/rzeppelin/DESCRIPTION</exclude>
<exclude>**/R/rzeppelin/NAMESPACE</exclude>
<!--End of mechanical R files-->
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/derby.log</exclude>
<exclude>**/metastore_db/</exclude>
<exclude>**/README.md</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>

31
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -73,10 +73,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
private String scriptPath;
boolean pythonscriptRunning = false;
private static final int MAX_TIMEOUT_SEC = 10;
private long pythonPid;
public PySparkInterpreter(Properties property) {
super(property);
pythonPid = -1;
try {
File scriptFile = File.createTempFile("zeppelin_pyspark-", ".py");
scriptPath = scriptFile.getAbsolutePath();
@ -200,7 +202,16 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
gatewayServer.start();
// Run python shell
CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python"));
// Choose python in the order of
// PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
String pythonExec = getProperty("zeppelin.pyspark.python");
if (System.getenv("PYSPARK_PYTHON") != null) {
pythonExec = System.getenv("PYSPARK_PYTHON");
}
if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
}
CommandLine cmd = CommandLine.parse(pythonExec);
cmd.addArgument(scriptPath, false);
cmd.addArgument(Integer.toString(port), false);
cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
@ -310,7 +321,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
boolean pythonScriptInitialized = false;
Integer pythonScriptInitializeNotifier = new Integer(0);
public void onPythonScriptInitialized() {
public void onPythonScriptInitialized(long pid) {
pythonPid = pid;
synchronized (pythonScriptInitializeNotifier) {
pythonScriptInitialized = true;
pythonScriptInitializeNotifier.notifyAll();
@ -411,10 +423,25 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public void interrupt() throws IOException {
if (pythonPid > -1) {
logger.info("Sending SIGINT signal to PID : " + pythonPid);
Runtime.getRuntime().exec("kill -SIGINT " + pythonPid);
} else {
logger.warn("Non UNIX/Linux system, close the interpreter");
close();
}
}
@Override
public void cancel(InterpreterContext context) {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
sparkInterpreter.cancel(context);
try {
interrupt();
} catch (IOException e) {
logger.error("Error", e);
}
}
@Override

12
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -38,6 +38,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.SparkEnv;
import org.apache.spark.SecurityManager;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.repl.SparkILoop;
import org.apache.spark.scheduler.ActiveJob;
import org.apache.spark.scheduler.DAGScheduler;
@ -126,6 +127,7 @@ public class SparkInterpreter extends Interpreter {
private SparkVersion sparkVersion;
private static File outputDir; // class outputdir for scala 2.11
private Object classServer; // classserver for scala 2.11
private JavaSparkContext jsc;
public SparkInterpreter(Properties property) {
@ -152,6 +154,15 @@ public class SparkInterpreter extends Interpreter {
}
}
public JavaSparkContext getJavaSparkContext() {
synchronized (sharedInterpreterLock) {
if (jsc == null) {
jsc = JavaSparkContext.fromSparkContext(sc);
}
return jsc;
}
}
public boolean isSparkContextInitialized() {
synchronized (sharedInterpreterLock) {
return sc != null;
@ -1422,6 +1433,7 @@ public class SparkInterpreter extends Interpreter {
}
sparkSession = null;
sc = null;
jsc = null;
if (classServer != null) {
Utils.invokeMethod(classServer, "stop");
classServer = null;

4
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkRBackend;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
@ -45,6 +46,7 @@ public class SparkRInterpreter extends Interpreter {
private SparkInterpreter sparkInterpreter;
private ZeppelinR zeppelinR;
private SparkContext sc;
private JavaSparkContext jsc;
public SparkRInterpreter(Properties property) {
super(property);
@ -73,8 +75,10 @@ public class SparkRInterpreter extends Interpreter {
this.sparkInterpreter = getSparkInterpreter();
this.sc = sparkInterpreter.getSparkContext();
this.jsc = sparkInterpreter.getJavaSparkContext();
SparkVersion sparkVersion = new SparkVersion(sc.version());
ZeppelinRContext.setSparkContext(sc);
ZeppelinRContext.setJavaSparkContext(jsc);
if (Utils.isSpark2()) {
ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession());
}

6
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.spark;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SQLContext;
/**
@ -28,6 +29,7 @@ public class ZeppelinRContext {
private static SQLContext sqlContext;
private static ZeppelinContext zeppelinContext;
private static Object sparkSession;
private static JavaSparkContext javaSparkContext;
public static void setSparkContext(SparkContext sparkContext) {
ZeppelinRContext.sparkContext = sparkContext;
@ -60,4 +62,8 @@ public class ZeppelinRContext {
public static Object getSparkSession() {
return sparkSession;
}
public static void setJavaSparkContext(JavaSparkContext jsc) { javaSparkContext = jsc; }
public static JavaSparkContext getJavaSparkContext() { return javaSparkContext; }
}

2
spark/src/main/resources/R/zeppelin_sparkr.R
View file

@ -45,7 +45,7 @@ assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
if (version >= 20000) {
assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv)
assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
assign(".sparkRjsc", get(".sc", envir = SparkR:::.sparkREnv), envir=SparkR:::.sparkREnv)
assign(".sparkRjsc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getJavaSparkContext"), envir = SparkR:::.sparkREnv)
}
assign(".sqlc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSqlContext"), envir = SparkR:::.sparkREnv)
assign("sqlContext", get(".sqlc", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)

5
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -83,6 +83,9 @@ class PyZeppelinContext(dict):
def get(self, key):
return self.__getitem__(key)
def getInterpreterContext(self):
return self.z.getInterpreterContext()
def input(self, name, defaultValue=""):
return self.z.input(name, defaultValue)
@ -252,7 +255,7 @@ java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
intp = gateway.entry_point
intp.onPythonScriptInitialized()
intp.onPythonScriptInitialized(os.getpid())
jsc = intp.getJavaSparkContext()

30
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java
View file

@ -33,6 +33,8 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.junit.Assert.*;
@ -120,4 +122,32 @@ public class PySparkInterpreterTest {
assertTrue(completions.size() > 0);
}
}
private class infinityPythonJob implements Runnable {
@Override
public void run() {
String code = "import time\nwhile True:\n time.sleep(1)" ;
InterpreterResult ret = pySparkInterpreter.interpret(code, context);
assertNotNull(ret);
Pattern expectedMessage = Pattern.compile("KeyboardInterrupt");
Matcher m = expectedMessage.matcher(ret.message().toString());
assertTrue(m.find());
}
}
@Test
public void testCancelIntp() throws InterruptedException {
if (getSparkVersionNumber() > 11) {
assertEquals(InterpreterResult.Code.SUCCESS,
pySparkInterpreter.interpret("a = 1\n", context).code());
Thread t = new Thread(new infinityPythonJob());
t.start();
Thread.sleep(5000);
pySparkInterpreter.cancel(context);
assertTrue(t.isAlive());
t.join(2000);
assertFalse(t.isAlive());
}
}
}

9
travis_check.py
View file

@ -54,8 +54,11 @@ def getBuildStatus(author, commit):
# get latest 25 builds
resp = requests.get(url=travisApi + "/repos/" + author + "/zeppelin/builds")
data = json.loads(resp.text)
build = None
if len(data) == 0:
return build;
for b in data:
if b["commit"][:len(commit)] == commit:
resp = requests.get(url=travisApi + "/repos/" + author + "/zeppelin/builds/" + str(b["id"]))
@ -103,8 +106,8 @@ for sleep in check:
info("Get build status ...")
build = getBuildStatus(author, commit)
if build == None:
info("Can't find build for commit= " + commit)
sys.exit(1)
info("Can't find build for commit " + commit + " from " + author)
sys.exit(2)
print("Build https://travis-ci.org/" + author + "/zeppelin/builds/" + str(build["id"]))
failure, running = printBuildStatus(build)

25
zeppelin-examples/zeppelin-example-spell-echo/index.js
View file

@ -26,7 +26,28 @@ export default class EchoSpell extends SpellBase {
super("%echo");
}
interpret(paragraphText) {
return new SpellResult(paragraphText);
/**
* Consumes text and return `SpellResult`.
*
* @param paragraphText {string} which doesn't include magic
* @param config {Object}
* @return {SpellResult}
*/
interpret(paragraphText, config) {
let repeat = 1;
try {
repeat = parseFloat(config.repeat);
} catch (error) {
/** ignore, use default value */
}
let repeated = "";
for (let i = 0; i < repeat; i++) {
repeated += `${paragraphText}\n`;
}
return new SpellResult(repeated);
}
}

7
zeppelin-examples/zeppelin-example-spell-echo/zeppelin-example-spell-echo.json
View file

@ -21,6 +21,13 @@
"artifact" : "./zeppelin-examples/zeppelin-example-spell-echo",
"license" : "Apache-2.0",
"icon" : "<i class='fa fa-repeat'></i>",
"config": {
"repeat": {
"type": "number",
"description": "How many times to repeat",
"defaultValue": 1
}
},
"spell": {
"magic": "%echo",
"usage": "%echo <TEXT>"

16
zeppelin-examples/zeppelin-example-spell-translator/index.js
View file

@ -28,11 +28,18 @@ export default class TranslatorSpell extends SpellBase {
super("%translator");
}
interpret(paragraphText) {
/**
* Consumes text and return `SpellResult`.
*
* @param paragraphText {string} which doesn't include magic
* @param config {Object}
* @return {SpellResult}
*/
interpret(paragraphText, config) {
const parsed = this.parseConfig(paragraphText);
const auth = config['access-token'];
const source = parsed.source;
const target = parsed.target;
const auth = parsed.auth;
const text = parsed.text;
/**
@ -49,7 +56,7 @@ export default class TranslatorSpell extends SpellBase {
}
parseConfig(text) {
const pattern = /^\s*(\S+)-(\S+)\s*(\S+)([\S\s]*)/g;
const pattern = /^\s*(\S+)-(\S+)\s*([\S\s]*)/g;
const match = pattern.exec(text);
if (!match) {
@ -59,8 +66,7 @@ export default class TranslatorSpell extends SpellBase {
return {
source: match[1],
target: match[2],
auth: match[3],
text: match[4],
text: match[3],
}
}

7
zeppelin-examples/zeppelin-example-spell-translator/zeppelin-example-spell-translator.json
View file

@ -21,6 +21,13 @@
"artifact" : "./zeppelin-examples/zeppelin-example-spell-translator",
"license" : "Apache-2.0",
"icon" : "<i class='fa fa-globe '></i>",
"config": {
"access-token": {
"type": "string",
"description": "access token for Google Translation API",
"defaultValue": "EXAMPLE-TOKEN"
}
},
"spell": {
"magic": "%translator",
"usage": "%translator <source>-<target> <access-key> <TEXT>"

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java
View file

@ -67,7 +67,15 @@ public class Booter {
}
public static RemoteRepository newCentralRepository() {
return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/");
String mvnRepo = System.getenv("ZEPPELIN_INTERPRETER_DEP_MVNREPO");
if (mvnRepo == null) {
mvnRepo = System.getProperty("zeppelin.interpreter.dep.mvnRepo");
}
if (mvnRepo == null) {
mvnRepo = "http://repo1.maven.org/maven2/";
}
return new RemoteRepository("central", "default", mvnRepo);
}
public static RemoteRepository newLocalRepository() {

24
zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java
View file

@ -18,6 +18,8 @@ package org.apache.zeppelin.helium;
import org.apache.zeppelin.annotation.Experimental;
import java.util.Map;
/**
* Helium package definition
*/
@ -32,8 +34,13 @@ public class HeliumPackage {
// [[ .. and .. and .. ] or [ .. and .. and ..] ..]
private String license;
private String icon;
private String published;
public SpellPackageInfo spell;
private String groupId; // get groupId of INTERPRETER type package
private String artifactId; // get artifactId of INTERPRETER type package
private SpellPackageInfo spell;
private Map<String, Object> config;
public HeliumPackage(HeliumType type,
String name,
@ -100,11 +107,26 @@ public class HeliumPackage {
public String getLicense() {
return license;
}
public String getIcon() {
return icon;
}
public String getPublishedDate() {
return published;
}
public String getGroupId() {
return groupId;
}
public String getArtifactId() {
return artifactId;
}
public SpellPackageInfo getSpellInfo() {
return spell;
}
public Map<String, Object> getConfig() { return config; }
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -192,4 +192,8 @@ public class InterpreterContext {
public void setRemoteWorksController(RemoteWorksController remoteWorksController) {
this.remoteWorksController = remoteWorksController;
}
public InterpreterOutput out() {
return out;
}
}

89
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -17,15 +17,21 @@
package org.apache.zeppelin.interpreter;
import java.util.*;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* InterpreterGroup is list of interpreters in the same interpreter group.
@ -43,7 +49,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
String id;
Logger LOGGER = Logger.getLogger(InterpreterGroup.class);
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
AngularObjectRegistry angularObjectRegistry;
InterpreterHookRegistry hookRegistry;
@ -165,47 +171,70 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
*/
public void close(String sessionId) {
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
List<Interpreter> intpForSession = this.get(sessionId);
close(intpForSession);
final List<Interpreter> intpForSession = this.get(sessionId);
if (remoteInterpreterProcess != null) {
remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
close(intpForSession);
}
private void close(Collection<Interpreter> intpToClose) {
private void close(final Collection<Interpreter> intpToClose) {
close(null, null, null, intpToClose);
}
public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey) {
close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
}
private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
if (intpToClose == null) {
return;
}
List<Thread> closeThreads = new LinkedList<>();
Thread t = new Thread() {
public void run() {
for (Interpreter interpreter : intpToClose) {
Scheduler scheduler = interpreter.getScheduler();
interpreter.close();
for (final Interpreter intp : intpToClose) {
Thread t = new Thread() {
public void run() {
Scheduler scheduler = intp.getScheduler();
intp.close();
if (scheduler != null) {
if (null != scheduler) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
}
};
t.start();
closeThreads.add(t);
}
if (remoteInterpreterProcess != null) {
//TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
// refernceCount is a proper value. And as the same reason, we must not call
// remoteInterpreterProcess.dereference twice - this method also be called by
// interpreter.close().
for (Thread t : closeThreads) {
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreter", e);
// remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
// TODO(jl): While closing interpreters in a same session, we should remove after all
// interpreters are removed. OMG. It's too dirty!!
if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
interpreterGroupRef.remove(processKey);
} else {
interpreterGroup.remove(sessionKey);
}
}
}
};
t.start();
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreter: {}", getId(), e);
}
}
/**

14
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java
View file

@ -21,6 +21,7 @@ import java.net.URL;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
@ -74,11 +75,11 @@ public class LazyOpenInterpreter
@Override
public void close() {
synchronized (intp) {
if (opened == true) {
intp.close();
opened = false;
}
// TODO(jl): Remove this trick!!
// intp.close() should be called to reduce referenceCount
if (isOpen() || intp instanceof RemoteInterpreter) {
intp.close();
opened = false;
}
}
@ -102,6 +103,9 @@ public class LazyOpenInterpreter
@Override
public FormType getFormType() {
// RemoteInterpreter's this method calls init() internally, and which cause to increase the
// number of referenceCount and it affects incorrectly
open();
return intp.getFormType();
}

15
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -26,7 +26,6 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
@ -64,6 +63,7 @@ public class RemoteInterpreter extends Interpreter {
private String userName;
private Boolean isUserImpersonate;
private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
private String interpreterGroupName;
/**
* Remote interpreter and manage interpreter process
@ -72,7 +72,7 @@ public class RemoteInterpreter extends Interpreter {
String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
int outputLimit) {
int outputLimit, String interpreterGroupName) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
@ -88,6 +88,7 @@ public class RemoteInterpreter extends Interpreter {
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
this.interpreterGroupName = interpreterGroupName;
}
@ -185,7 +186,7 @@ public class RemoteInterpreter extends Interpreter {
// create new remote process
remoteProcess = new RemoteInterpreterManagedProcess(
interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
remoteInterpreterProcessListener, applicationEventListener);
remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
}
intpGroup.setRemoteInterpreterProcess(remoteProcess);
@ -251,6 +252,14 @@ public class RemoteInterpreter extends Interpreter {
synchronized (interpreterGroup) {
// initialize all interpreters in this interpreter group
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
// TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
// initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
// doesn't call open method, it's not open. It causes problem while running intp.close()
// In case of Spark, this method initializes all of interpreters and init() method increases
// reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
// other interpreters doesn't do anything because those LazyInterpreters aren't open.
// But for now, we have to initialise all of interpreters for some reasons.
// See Interpreter.getInterpreterInTheSameSessionByClassName(String)
for (Interpreter intp : new ArrayList<>(interpreters)) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
View file

@ -44,6 +44,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
private int port = -1;
private final String interpreterDir;
private final String localRepoDir;
private final String interpreterGroupName;
private Map<String, String> env;
@ -54,14 +55,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener,
ApplicationEventListener appListener) {
ApplicationEventListener appListener,
String interpreterGroupName) {
super(new RemoteInterpreterEventPoller(listener, appListener),
connectTimeout);
this.interpreterRunner = intpRunner;
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
this.interpreterGroupName = interpreterGroupName;
}
RemoteInterpreterManagedProcess(String intpRunner,
@ -69,13 +71,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
String localRepoDir,
Map<String, String> env,
RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
int connectTimeout,
String interpreterGroupName) {
super(remoteInterpreterEventPoller,
connectTimeout);
this.interpreterRunner = intpRunner;
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
this.interpreterGroupName = interpreterGroupName;
}
@Override
@ -108,6 +112,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
}
cmdLine.addArgument("-l", false);
cmdLine.addArgument(localRepoDir, false);
cmdLine.addArgument("-n", false);
cmdLine.addArgument(interpreterGroupName, false);
executor = new DefaultExecutor();

21
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -276,17 +276,18 @@ public class RemoteInterpreterServer
}
// close interpreters
List<Interpreter> interpreters;
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
while (it.hasNext()) {
Interpreter inp = it.next();
if (inp.getClassName().equals(className)) {
inp.close();
it.remove();
break;
}
interpreters = interpreterGroup.get(sessionKey);
}
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
while (it.hasNext()) {
Interpreter inp = it.next();
if (inp.getClassName().equals(className)) {
inp.close();
it.remove();
break;
}
}
}

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -182,22 +182,16 @@ public abstract class Job {
this.exception = null;
errorMessage = null;
dateFinished = new Date();
progressUpdator.terminate();
} catch (NullPointerException e) {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} catch (Throwable e) {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} finally {
if (progressUpdator != null) {
progressUpdator.interrupt();
}
//aborted = false;
}
}

49
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java
View file

@ -21,48 +21,45 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Polls job progress with given interval
*
* @see Job#progress()
* @see JobListener#onProgressUpdate(org.apache.zeppelin.scheduler.Job, int)
*
* TODO(moon) : add description.
*/
public class JobProgressPoller extends Thread {
public static final long DEFAULT_INTERVAL_MSEC = 500;
Logger logger = LoggerFactory.getLogger(JobProgressPoller.class);
private static final Logger logger = LoggerFactory.getLogger(JobProgressPoller.class);
private Job job;
private long intervalMs;
boolean terminate = false;
public JobProgressPoller(Job job, long intervalMs) {
super("JobProgressPoller, jobId=" + job.getId());
this.job = job;
this.intervalMs = intervalMs;
if (intervalMs < 0) {
throw new IllegalArgumentException("polling interval can't be " + intervalMs);
}
this.intervalMs = intervalMs == 0 ? DEFAULT_INTERVAL_MSEC : intervalMs;
}
@Override
public void run() {
if (intervalMs < 0) {
return;
} else if (intervalMs == 0) {
intervalMs = DEFAULT_INTERVAL_MSEC;
}
while (terminate == false) {
JobListener listener = job.getListener();
if (listener != null) {
try {
if (job.isRunning()) {
listener.onProgressUpdate(job, job.progress());
try {
while (!Thread.interrupted()) {
JobListener listener = job.getListener();
if (listener != null) {
try {
if (job.isRunning()) {
listener.onProgressUpdate(job, job.progress());
}
} catch (Exception e) {
logger.error("Can not get or update progress", e);
}
} catch (Exception e) {
logger.error("Can not get or update progress", e);
}
}
try {
Thread.sleep(intervalMs);
} catch (InterruptedException e) {
logger.error("Exception in JobProgressPoller while run Thread.sleep", e);
}
}
}
public void terminate() {
terminate = true;
} catch (InterruptedException ignored) {}
}
}

39
zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/HeliumPackageTest.java
View file

@ -20,6 +20,8 @@ package org.apache.zeppelin.helium;
import com.google.gson.Gson;
import org.junit.Test;
import java.util.Map;
import static org.junit.Assert.*;
public class HeliumPackageTest {
@ -28,7 +30,7 @@ public class HeliumPackageTest {
@Test
public void parseSpellPackageInfo() {
String exampleSpell = "{\n" +
String examplePackage = "{\n" +
" \"type\" : \"SPELL\",\n" +
" \"name\" : \"echo-spell\",\n" +
" \"description\" : \"'%echo' - return just what receive (example)\",\n" +
@ -41,8 +43,41 @@ public class HeliumPackageTest {
" }\n" +
"}";
HeliumPackage p = gson.fromJson(exampleSpell, HeliumPackage.class);
HeliumPackage p = gson.fromJson(examplePackage, HeliumPackage.class);
assertEquals(p.getSpellInfo().getMagic(), "%echo");
assertEquals(p.getSpellInfo().getUsage(), "%echo <TEXT>");
}
@Test
public void parseConfig() {
String examplePackage = "{\n" +
" \"type\" : \"SPELL\",\n" +
" \"name\" : \"translator-spell\",\n" +
" \"description\" : \"Translate langauges using Google API (examaple)\",\n" +
" \"artifact\" : \"./zeppelin-examples/zeppelin-example-spell-translator\",\n" +
" \"license\" : \"Apache-2.0\",\n" +
" \"icon\" : \"<i class='fa fa-globe '></i>\",\n" +
" \"config\": {\n" +
" \"access-token\": {\n" +
" \"type\": \"string\",\n" +
" \"description\": \"access token for Google Translation API\",\n" +
" \"defaultValue\": \"EXAMPLE-TOKEN\"\n" +
" }\n" +
" },\n" +
" \"spell\": {\n" +
" \"magic\": \"%translator\",\n" +
" \"usage\": \"%translator <source>-<target> <access-key> <TEXT>\"\n" +
" }\n" +
"}";
HeliumPackage p = gson.fromJson(examplePackage, HeliumPackage.class);
Map<String, Object> config = p.getConfig();
Map<String, Object> accessToken = (Map<String, Object>) config.get("access-token");
assertEquals((String) accessToken.get("type"),"string");
assertEquals((String) accessToken.get("description"),
"access token for Google Translation API");
assertEquals((String) accessToken.get("defaultValue"),
"EXAMPLE-TOKEN");
}
}

9
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
View file

@ -44,7 +44,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null);
10 * 1000, null, null,"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@ -61,7 +61,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class), 10 * 1000);
mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
rip.reference(intpGroup, "anonymous", false);
assertEquals(0, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
@ -104,7 +104,8 @@ public class RemoteInterpreterProcessTest {
"fakeRepo",
new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class)
, 10 * 1000);
, 10 * 1000,
"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@ -117,7 +118,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
"echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null);
10 * 1000, null, null, "fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
try {

58
zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java
View file

@ -36,7 +36,6 @@ import org.apache.shiro.realm.ldap.LdapUtils;
import org.apache.shiro.subject.MutablePrincipalCollection;
import org.apache.shiro.subject.PrincipalCollection;
import org.apache.shiro.util.StringUtils;
import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -45,7 +44,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@ -107,6 +105,8 @@ import javax.naming.ldap.PagedResultsControl;
* apache,dc=org
* ldapRealm.contextFactory.systemPassword=S{ALIAS=ldcSystemPassword} [urls]
* **=authcBasic
* # enable support for nested groups using the LDAP_MATCHING_RULE_IN_CHAIN operator
* ldapRealm.groupSearchEnableMatchingRuleInChain = true
*
* <p># optional mapping from physical groups to logical application roles
* ldapRealm.rolesByGroup = \ LDN_USERS: user_role,\ NYK_USERS: user_role,\
@ -128,6 +128,12 @@ public class LdapRealm extends JndiLdapRealm {
private static final String SUBJECT_USER_GROUPS = "subject.userGroups";
private static final String MEMBER_URL = "memberUrl";
private static final String POSIX_GROUP = "posixGroup";
// LDAP Operator '1.2.840.113556.1.4.1941'
// walks the chain of ancestry in objects all the way to the root until it finds a match
// see https://msdn.microsoft.com/en-us/library/aa746475(v=vs.85).aspx
private static final String MATCHING_RULE_IN_CHAIN_FORMAT =
"(&(objectClass=%s)(%s:1.2.840.113556.1.4.1941:=%s))";
private static Pattern TEMPLATE_PATTERN = Pattern.compile("\\{(\\d+?)\\}");
private static String DEFAULT_PRINCIPAL_REGEX = "(.*)";
@ -153,13 +159,14 @@ public class LdapRealm extends JndiLdapRealm {
private String userSearchAttributeTemplate = "{0}";
private String userSearchScope = "subtree";
private String groupSearchScope = "subtree";
private boolean groupSearchEnableMatchingRuleInChain;
private String groupSearchBase;
private String groupObjectClass = "groupOfNames";
// typical value: member, uniqueMember, meberUrl
// typical value: member, uniqueMember, memberUrl
private String memberAttribute = "member";
private String groupIdAttribute = "cn";
@ -282,16 +289,38 @@ public class LdapRealm extends JndiLdapRealm {
NamingEnumeration<SearchResult> searchResultEnum = null;
SearchControls searchControls = getGroupSearchControls();
try {
searchResultEnum = ldapCtx.search(
if (groupSearchEnableMatchingRuleInChain) {
searchResultEnum = ldapCtx.search(
getGroupSearchBase(),
String.format(
MATCHING_RULE_IN_CHAIN_FORMAT, groupObjectClass, memberAttribute, userDn),
searchControls);
while (searchResultEnum != null && searchResultEnum.hasMore()) {
// searchResults contains all the groups in search scope
numResults++;
final SearchResult group = searchResultEnum.next();
Attribute attribute = group.getAttributes().get(getGroupIdAttribute());
String groupName = attribute.get().toString();
String roleName = roleNameFor(groupName);
if (roleName != null) {
roleNames.add(roleName);
} else {
roleNames.add(groupName);
}
}
} else {
searchResultEnum = ldapCtx.search(
getGroupSearchBase(),
"objectClass=" + groupObjectClass,
searchControls);
while (searchResultEnum != null && searchResultEnum.hasMore()) {
// searchResults contains all the groups in search scope
numResults++;
final SearchResult group = searchResultEnum.next();
addRoleIfMember(userDn, group, roleNames, groupNames, ldapContextFactory);
while (searchResultEnum != null && searchResultEnum.hasMore()) {
// searchResults contains all the groups in search scope
numResults++;
final SearchResult group = searchResultEnum.next();
addRoleIfMember(userDn, group, roleNames, groupNames, ldapContextFactory);
}
}
} catch (PartialResultException e) {
log.debug("Ignoring PartitalResultException");
@ -682,6 +711,15 @@ public class LdapRealm extends JndiLdapRealm {
public void setGroupSearchScope(final String scope) {
this.groupSearchScope = (scope == null ? null : scope.trim().toLowerCase());
}
public boolean isGroupSearchEnableMatchingRuleInChain() {
return groupSearchEnableMatchingRuleInChain;
}
public void setGroupSearchEnableMatchingRuleInChain(
boolean groupSearchEnableMatchingRuleInChain) {
this.groupSearchEnableMatchingRuleInChain = groupSearchEnableMatchingRuleInChain;
}
private SearchControls getUserSearchControls() {
SearchControls searchControls = SUBTREE_SCOPE;

126
zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
View file

@ -18,8 +18,10 @@
package org.apache.zeppelin.rest;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.notebook.Note;
@ -34,6 +36,7 @@ import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* Helium Rest Api
@ -56,13 +59,34 @@ public class HeliumRestApi {
}
/**
* Get all packages
* @return
* Get all package infos
*/
@GET
@Path("all")
public Response getAll() {
return new JsonResponse(Response.Status.OK, "", helium.getAllPackageInfo()).build();
@Path("package")
public Response getAllPackageInfo() {
return new JsonResponse(
Response.Status.OK, "", helium.getAllPackageInfo()).build();
}
/**
* Get single package info
*/
@GET
@Path("package/{packageName}")
public Response getSinglePackageInfo(@PathParam("packageName") String packageName) {
if (StringUtils.isEmpty(packageName)) {
return new JsonResponse(
Response.Status.BAD_REQUEST,
"Can't get package info for empty name").build();
}
try {
return new JsonResponse(
Response.Status.OK, "", helium.getSinglePackageInfo(packageName)).build();
} catch (RuntimeException e) {
logger.error(e.getMessage(), e);
return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
}
}
@GET
@ -165,6 +189,98 @@ public class HeliumRestApi {
return new JsonResponse(Response.Status.OK, order).build();
}
@GET
@Path("spell/config/{packageName}")
public Response getSpellConfigUsingMagic(@PathParam("packageName") String packageName) {
if (StringUtils.isEmpty(packageName)) {
return new JsonResponse(Response.Status.BAD_REQUEST,
"packageName is empty" ).build();
}
try {
Map<String, Map<String, Object>> config =
helium.getSpellConfig(packageName);
if (config == null) {
return new JsonResponse(Response.Status.BAD_REQUEST,
"Failed to find enabled package for " + packageName).build();
}
return new JsonResponse(Response.Status.OK, config).build();
} catch (RuntimeException e) {
logger.error(e.getMessage(), e);
return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
}
}
@GET
@Path("config")
public Response getAllPackageConfigs() {
try {
Map<String, Map<String, Object>> config = helium.getAllPackageConfig();
return new JsonResponse(Response.Status.OK, config).build();
} catch (RuntimeException e) {
logger.error(e.getMessage(), e);
return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
}
}
@GET
@Path("config/{packageName}/{artifact}")
public Response getPackageConfig(@PathParam("packageName") String packageName,
@PathParam("artifact") String artifact) {
if (StringUtils.isEmpty(packageName) || StringUtils.isEmpty(artifact)) {
return new JsonResponse(Response.Status.BAD_REQUEST,
"package name or artifact is empty"
).build();
}
try {
Map<String, Map<String, Object>> config =
helium.getPackageConfig(packageName, artifact);
if (config == null) {
return new JsonResponse(Response.Status.BAD_REQUEST,
"Failed to find package for " + artifact).build();
}
return new JsonResponse(Response.Status.OK, config).build();
} catch (RuntimeException e) {
logger.error(e.getMessage(), e);
return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()).build();
}
}
@POST
@Path("config/{packageName}/{artifact}")
public Response updatePackageConfig(@PathParam("packageName") String packageName,
@PathParam("artifact") String artifact,
String rawConfig) {
if (StringUtils.isEmpty(packageName) || StringUtils.isEmpty(artifact)) {
return new JsonResponse(Response.Status.BAD_REQUEST,
"package name or artifact is empty"
).build();
}
Map<String, Object> packageConfig = null;
try {
packageConfig = gson.fromJson(
rawConfig, new TypeToken<Map<String, Object>>(){}.getType());
helium.updatePackageConfig(artifact, packageConfig);
} catch (JsonParseException e) {
logger.error(e.getMessage(), e);
return new JsonResponse(Response.Status.BAD_REQUEST,
e.getMessage()).build();
} catch (IOException | RuntimeException e) {
return new JsonResponse(Response.Status.INTERNAL_SERVER_ERROR,
e.getMessage()).build();
}
return new JsonResponse(Response.Status.OK, packageConfig).build();
}
@POST
@Path("order/visualization")
public Response getVisualizationPackageOrder(String orderedPackageNameList) {

6
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -190,7 +190,11 @@ public class InterpreterRestApi {
RestartInterpreterRequest request = gson.fromJson(message, RestartInterpreterRequest.class);
String noteId = request == null ? null : request.getNoteId();
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
if (null == noteId) {
interpreterSettingManager.close(setting);
} else {
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
}
notebookServer.clearParagraphRuntimeInfo(setting);
} catch (InterpreterException e) {

4
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -126,7 +126,7 @@ public class ZeppelinServer extends Application {
new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
}
this.helium = new Helium(
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
@ -177,7 +177,7 @@ public class ZeppelinServer extends Application {
// Web UI
final WebAppContext webApp = setupWebAppContext(contexts, conf);
// REST api
// Create `ZeppelinServer` using reflection and setup REST Api
setupRestApiContextHandler(webApp, conf);
// Notebook server

72
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -193,7 +193,8 @@ public class NotebookServer extends WebSocketServlet
}
String ticket = TicketContainer.instance.getTicket(messagereceived.principal);
if (ticket != null && !ticket.equals(messagereceived.ticket)) {
if (ticket != null &&
(messagereceived.ticket == null || !ticket.equals(messagereceived.ticket))) {
/* not to pollute logs, log instead of exception */
if (StringUtils.isEmpty(messagereceived.ticket)) {
LOG.debug("{} message: invalid ticket {} != {}", messagereceived.op,
@ -1081,6 +1082,7 @@ public class NotebookServer extends WebSocketServlet
if (note != null && !note.isTrash()){
fromMessage.put("name", Folder.TRASH_FOLDER_ID + "/" + note.getName());
renameNote(conn, userAndRoles, notebook, fromMessage, "move");
notebook.moveNoteToTrash(note.getId());
}
}
@ -1184,15 +1186,21 @@ public class NotebookServer extends WebSocketServlet
final Note note = notebook.getNote(noteId);
Paragraph p = note.getParagraph(paragraphId);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (note.isPersonalizedMode()) {
p = p.getUserParagraphMap().get(subject.getUser());
}
p.settings.setParams(params);
p.setConfig(config);
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (note.isPersonalizedMode()) {
p = p.getUserParagraph(subject.getUser());
p.settings.setParams(params);
p.setConfig(config);
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
}
note.persist(subject);
if (note.isPersonalizedMode()) {
@ -1759,6 +1767,15 @@ public class NotebookServer extends WebSocketServlet
p.settings.setParams(params);
p.setConfig(config);
if (note.isPersonalizedMode()) {
p = note.getParagraph(paragraphId);
p.setText(text);
p.setTitle(title);
p.setAuthenticationInfo(subject);
p.settings.setParams(params);
p.setConfig(config);
}
return p;
}
@ -1877,7 +1894,15 @@ public class NotebookServer extends WebSocketServlet
InterpreterResult.Type type, String output) {
Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT).put("noteId", noteId)
.put("paragraphId", paragraphId).put("index", index).put("type", type).put("data", output);
broadcast(noteId, msg);
Note note = notebook().getNote(noteId);
if (note.isPersonalizedMode()) {
String user = note.getParagraph(paragraphId).getUser();
if (null != user) {
multicastToUser(user, msg);
}
} else {
broadcast(noteId, msg);
}
}
@ -2115,7 +2140,7 @@ public class NotebookServer extends WebSocketServlet
@Override
public void onProgressUpdate(Job job, int progress) {
notebookServer.broadcast(note.getId(),
new Message(OP.PROGRESS).put("id", job.getId()).put("progress", job.progress()));
new Message(OP.PROGRESS).put("id", job.getId()).put("progress", progress));
}
@Override
@ -2146,7 +2171,9 @@ public class NotebookServer extends WebSocketServlet
}
}
if (job instanceof Paragraph) {
notebookServer.broadcastParagraph(note, (Paragraph) job);
Paragraph p = (Paragraph) job;
p.setStatusToUserParagraph(job.getStatus());
notebookServer.broadcastParagraph(note, p);
}
try {
notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
@ -2281,7 +2308,6 @@ public class NotebookServer extends WebSocketServlet
resp.put("editor", notebook().getInterpreterSettingManager().
getEditorSetting(interpreter, user, noteId, replName));
conn.send(serializeMessage(resp));
return;
}
private void getInterpreterSettings(NotebookSocket conn, AuthenticationInfo subject)
@ -2324,11 +2350,31 @@ public class NotebookServer extends WebSocketServlet
.equals(WatcherSecurityKey.getKey()));
}
/**
* Send websocket message to all connections regardless of notebook id
*/
private void broadcastToAllConnections(String serialized) {
broadcastToAllConnectionsExcept(null, serialized);
}
private void broadcastToAllConnectionsExcept(NotebookSocket exclude, String serialized) {
synchronized (connectedSockets) {
for (NotebookSocket conn: connectedSockets) {
if (exclude != null && exclude.equals(conn)) {
continue;
}
try {
conn.send(serialized);
} catch (IOException e) {
LOG.error("Cannot broadcast message to watcher", e);
}
}
}
}
private void broadcastToWatchers(String noteId, String subject, Message message) {
synchronized (watcherSockets) {
if (watcherSockets.isEmpty()) {
return;
}
for (NotebookSocket watcher : watcherSockets) {
try {
watcher.send(

29
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -256,6 +256,35 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
}
@Test
public void testRunOnSelectionCheckbox() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
String xpathToCheckbox = getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-checked, 'true')]";
createNewNote();
waitForParagraph(1, "READY");
setTextOfParagraph(1, "%md My selection is ${my selection=1,1|2|3}");
runParagraph(1);
driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-settings']")).click();
collector.checkThat("'Run on selection change' checkbox will be shown under dropdown menu ",
driver.findElement(By.xpath(getParagraphXPath(1) + "//ul/li/form/input[contains(@ng-click, 'turnOnAutoRun(paragraph)')]")).isDisplayed(),
CoreMatchers.equalTo(true));
driver.findElement(By.xpath(xpathToCheckbox)).click();
collector.checkThat("If 'Run on selection change' checkbox is unchecked, 'paragraph.config.runOnSelectionChange' will be false ",
driver.findElement(By.xpath(getParagraphXPath(1) + "//ul/li/span[contains(@ng-if, 'paragraph.config.runOnSelectionChange == false')]")).isDisplayed(),
CoreMatchers.equalTo(true));
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testRunOnSelectionButton ", e);
}
}
@Test
public void testClearOutputButton() throws Exception {
if (!endToEndTestEnabled()) {

3
zeppelin-web/README.md
View file

@ -31,6 +31,9 @@ $ yarn run build
# you need to run zeppelin backend instance also
$ yarn run dev
# If you are using a custom port, you must use the 'SERVER_PORT' variable to run the web application development mode
$ SERVER_PORT=8080 yarn run dev
# execute tests
$ yarn run test
```

2
zeppelin-web/bower.json
View file

@ -17,7 +17,7 @@
"ace-builds": "1.2.6",
"angular-ui-ace": "0.1.3",
"jquery.scrollTo": "~1.4.13",
"nvd3": "~1.7.1",
"nvd3": "~1.8.5",
"angular-dragdrop": "~1.0.8",
"perfect-scrollbar": "~0.5.4",
"ng-sortable": "~1.3.3",

46
zeppelin-web/pom.xml
View file

@ -58,52 +58,6 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
<exclude>.gitignore</exclude>
<exclude>.babelrc</exclude>
<exclude>.bowerrc</exclude>
<exclude>.editorconfig</exclude>
<exclude>.jscsrc</exclude>
<exclude>.eslintrc</exclude>
<exclude>.tmp/**</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>node/**</exclude>
<exclude>node_modules/**</exclude>
<exclude>bower_components/**</exclude>
<exclude>src/**/*.test.js</exclude>
<exclude>reports/**</exclude>
<exclude>dist/**</exclude>
<exclude>src/.buildignore</exclude>
<exclude>src/fonts/fontawesome*</exclude>
<exclude>src/fonts/font-awesome*</exclude>
<exclude>src/styles/font-awesome*</exclude>
<exclude>src/fonts/Simple-Line*</exclude>
<exclude>src/fonts/simple-line*</exclude>
<exclude>src/fonts/Patua-One*</exclude>
<exclude>src/fonts/patua-one*</exclude>
<exclude>src/fonts/Roboto*</exclude>
<exclude>src/fonts/roboto*</exclude>
<exclude>src/fonts/Source-Code-Pro*</exclude>
<exclude>src/fonts/source-code-pro*</exclude>
<exclude>bower.json</exclude>
<exclude>**/package.json</exclude>
<exclude>**/.npmignore</exclude>
<exclude>yarn.lock</exclude>
<exclude>*.md</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>

100
zeppelin-web/src/app/helium/helium.config.js Normal file
View file

@ -0,0 +1,100 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const HeliumConfFieldType = {
NUMBER: 'number',
JSON: 'json',
STRING: 'string',
};
/**
* @param persisted <Object> including `type`, `description`, `defaultValue` for each conf key
* @param spec <Object> including `value` for each conf key
*/
export function mergePersistedConfWithSpec(persisted, spec) {
const confs = [];
for(let name in spec) {
const specField = spec[name];
const persistedValue = persisted[name];
const value = (persistedValue) ? persistedValue : specField.defaultValue;
const merged = {
name: name, type: specField.type, description: specField.description,
value: value, defaultValue: specField.defaultValue,
};
confs.push(merged);
}
return confs;
}
export function createPackageConf(defaultPackages, persistedPackacgeConfs) {
let packageConfs = {};
for (let name in defaultPackages) {
const pkgInfo = defaultPackages[name];
const configSpec = pkgInfo.pkg.config;
if (!configSpec) { continue; }
const version = pkgInfo.pkg.version;
if (!version) { continue; }
let config = {};
if (persistedPackacgeConfs[name] && persistedPackacgeConfs[name][version]) {
config = persistedPackacgeConfs[name][version];
}
const confs = mergePersistedConfWithSpec(config, configSpec);
packageConfs[name] = confs;
}
return packageConfs;
}
export function parseConfigValue(type, stringified) {
let value = stringified;
try {
if (HeliumConfFieldType.NUMBER === type) {
value = parseFloat(stringified);
} else if (HeliumConfFieldType.JSON === type) {
value = JSON.parse(stringified);
}
} catch(error) {
// return just the stringified one
console.error(`Failed to parse conf type ${type}, value ${value}`);
}
return value;
}
/**
* create persistable config object
*/
export function createPersistableConfig(currentConf) {
// persist key-value only
// since other info (e.g type, desc) can be provided by default config
const filtered = currentConf.reduce((acc, c) => {
let value = parseConfigValue(c.type, c.value);
acc[c.name] = value;
return acc;
}, {});
return filtered;
}

266
zeppelin-web/src/app/helium/helium.controller.js
View file

@ -14,60 +14,76 @@
import { HeliumType, } from '../../components/helium/helium-type';
angular.module('zeppelinWebApp').controller('HeliumCtrl', HeliumCtrl);
function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService) {
export default function HeliumCtrl($scope, $rootScope, $sce,
baseUrlSrv, ngToast, heliumService) {
'ngInject';
$scope.packageInfos = {};
$scope.defaultVersions = {};
$scope.pkgSearchResults = {};
$scope.defaultPackages = {};
$scope.showVersions = {};
$scope.bundleOrder = [];
$scope.bundleOrderChanged = false;
$scope.vizTypePkg = {}
$scope.spellTypePkg = {}
$scope.intpTypePkg = {}
$scope.appTypePkg = {}
$scope.numberOfEachPackageByType = {}
$scope.allPackageTypes = [HeliumType][0]
$scope.pkgListByType = 'VISUALIZATION'
$scope.defaultPackageConfigs = {}; // { pkgName, [{name, type, desc, value, defaultValue}] }
$scope.intpDefaultIcon = $sce.trustAsHtml('<img src="../assets/images/maven_default_icon.png" style="width: 12px"/>');
function init() {
// get all package info and set config
heliumService.getAllPackageInfoAndDefaultPackages()
.then(({ pkgSearchResults, defaultPackages }) => {
$scope.pkgSearchResults = pkgSearchResults;
$scope.defaultPackages = defaultPackages;
classifyPkgType($scope.defaultPackages)
return heliumService.getAllPackageConfigs()
})
.then(defaultPackageConfigs => {
$scope.defaultPackageConfigs = defaultPackageConfigs;
});
// 2. get vis package order
heliumService.getVisualizationPackageOrder()
.then(visPackageOrder => {
$scope.bundleOrder = visPackageOrder;
$scope.bundleOrderChanged = false;
});
}
var classifyPkgType = function(packageInfos) {
var vizTypePkg = {}
var spellTypePkg = {}
var intpTypePkg = {}
var appTypePkg = {}
var buildDefaultVersionListToDisplay = function(packageInfos) {
var defaultVersions = {};
// show enabled version if any version of package is enabled
for (var name in packageInfos) {
var pkgs = packageInfos[name];
for (var pkgIdx in pkgs) {
var pkg = pkgs[pkgIdx];
pkg.pkg.icon = $sce.trustAsHtml(pkg.pkg.icon);
if (pkg.enabled) {
defaultVersions[name] = pkg;
pkgs.splice(pkgIdx, 1);
break;
}
}
var pkgs = packageInfos[name]
var pkgType = pkgs.pkg.type
// show first available version if package is not enabled
if (!defaultVersions[name]) {
defaultVersions[name] = pkgs[0];
pkgs.splice(0, 1);
switch (pkgType) {
case HeliumType.VISUALIZATION:
vizTypePkg[name] = pkgs;
break;
case HeliumType.SPELL:
spellTypePkg[name] = pkgs;
break;
case HeliumType.INTERPRETER:
intpTypePkg[name] = pkgs;
break;
case HeliumType.APPLICATION:
appTypePkg[name] = pkgs;
break;
}
}
$scope.defaultVersions = defaultVersions;
};
var getAllPackageInfo = function() {
heliumService.getAllPackageInfo().
success(function(data, status) {
$scope.packageInfos = data.body;
buildDefaultVersionListToDisplay($scope.packageInfos);
}).
error(function(data, status) {
console.log('Can not load package info %o %o', status, data);
});
};
var getBundleOrder = function() {
heliumService.getVisualizationPackageOrder().
success(function(data, status) {
$scope.bundleOrder = data.body;
}).
error(function(data, status) {
console.log('Can not get bundle order %o %o', status, data);
});
$scope.vizTypePkg = vizTypePkg
$scope.spellTypePkg = spellTypePkg
$scope.appTypePkg = appTypePkg
$scope.intpTypePkg = intpTypePkg
};
$scope.bundleOrderListeners = {
@ -78,14 +94,6 @@ function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService
}
};
var init = function() {
getAllPackageInfo();
getBundleOrder();
$scope.bundleOrderChanged = false;
};
init();
$scope.saveBundleOrder = function() {
var confirm = BootstrapDialog.confirm({
closable: false,
@ -115,24 +123,24 @@ function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService
}
}
});
}
};
var getLicense = function(name, artifact) {
var pkg = _.filter($scope.defaultVersions[name], function(p) {
var filteredPkgSearchResults = _.filter($scope.defaultPackages[name], function(p) {
return p.artifact === artifact;
});
var license;
if (pkg.length === 0) {
pkg = _.filter($scope.packageInfos[name], function(p) {
if (filteredPkgSearchResults.length === 0) {
filteredPkgSearchResults = _.filter($scope.pkgSearchResults[name], function(p) {
return p.pkg.artifact === artifact;
});
if (pkg.length > 0) {
license = pkg[0].pkg.license;
if (filteredPkgSearchResults.length > 0) {
license = filteredPkgSearchResults[0].pkg.license;
}
} else {
license = pkg[0].license;
license = filteredPkgSearchResults[0].license;
}
if (!license) {
@ -141,40 +149,58 @@ function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService
return license;
}
$scope.enable = function(name, artifact) {
$scope.enable = function(name, artifact, type, groupId) {
var license = getLicense(name, artifact);
var confirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
var mavenArtifactInfoToHTML = groupId +':'+ artifact.split('@')[0] + ':' + artifact.split('@')[1];
var zeppelinVersion = $rootScope.zeppelinVersion;
var url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html';
var confirm = ''
if (type === 'INTERPRETER') {
confirm = BootstrapDialog.show({
title: '',
message: 'Do you want to enable ' + name + '?' +
'<div style="color:gray">' + artifact + '</div>' +
'<div style="border-top: 1px solid #efefef; margin-top: 10px; padding-top: 5px;">License</div>' +
'<div style="color:gray">' + license + '</div>',
callback: function(result) {
if (result) {
confirm.$modalFooter.find('button').addClass('disabled');
confirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Enabling');
heliumService.enable(name, artifact).
success(function(data, status) {
init();
confirm.close();
}).
error(function(data, status) {
confirm.close();
console.log('Failed to enable package %o %o. %o', name, artifact, data);
BootstrapDialog.show({
title: 'Error on enabling ' + name,
message: data.message
});
});
return false;
}
}
message: '<p>Below command will download maven artifact ' +
'<code style="font-size: 11.5px; background-color: #f5f5f5; color: #0a0a0a">' +
mavenArtifactInfoToHTML + '</code>' +
' and all of its transitive dependencies into interpreter/interpreter-name directory.<p>' +
'<div class="highlight"><pre><code class="text language-text" data-lang="text" style="font-size: 11.5px">' +
'./bin/install-interpreter.sh --name "interpreter-name" --artifact ' +
mavenArtifactInfoToHTML +' </code></pre>' +
'<p>After restart Zeppelin, create interpreter setting and bind it with your note. ' +
'For more detailed information, see <a target="_blank" href=' +
url + '>Interpreter Installation.</a></p>'
});
} else {
confirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
title: '',
message: 'Do you want to enable ' + name + '?' +
'<div style="color:gray">' + artifact + '</div>' +
'<div style="border-top: 1px solid #efefef; margin-top: 10px; padding-top: 5px;">License</div>' +
'<div style="color:gray">' + license + '</div>',
callback: function (result) {
if (result) {
confirm.$modalFooter.find('button').addClass('disabled');
confirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Enabling');
heliumService.enable(name, artifact, type).success(function (data, status) {
init();
confirm.close();
}).error(function (data, status) {
confirm.close();
console.log('Failed to enable package %o %o. %o', name, artifact, data);
BootstrapDialog.show({
title: 'Error on enabling ' + name,
message: data.message
});
});
return false;
}
}
});
}
};
$scope.disable = function(name) {
@ -226,4 +252,66 @@ function HeliumCtrl($scope, $rootScope, $sce, baseUrlSrv, ngToast, heliumService
return (pkg.type === HeliumType.SPELL || pkg.type === HeliumType.VISUALIZATION) &&
!$scope.isLocalPackage(pkgSearchResult);
};
$scope.hasMavenLink = function(pkgSearchResult) {
const pkg = pkgSearchResult.pkg;
return (pkg.type === HeliumType.APPLICATION || pkg.type === HeliumType.INTERPRETER) &&
!$scope.isLocalPackage(pkgSearchResult);
};
$scope.getPackageSize = function(pkgSearchResult, targetPkgType) {
var result = []
_.map(pkgSearchResult, function (pkg) {
result.push(_.find(pkg, {type: targetPkgType}))
})
return _.compact(result).length
}
$scope.configExists = function(pkgSearchResult) {
// helium package config is persisted per version
return pkgSearchResult.pkg.config && pkgSearchResult.pkg.artifact;
};
$scope.configOpened = function(pkgSearchResult) {
return pkgSearchResult.configOpened && !pkgSearchResult.configFetching;
};
$scope.getConfigButtonClass = function(pkgSearchResult) {
return (pkgSearchResult.configOpened && pkgSearchResult.configFetching) ?
'disabled' : '';
}
$scope.toggleConfigButton = function(pkgSearchResult) {
if (pkgSearchResult.configOpened) {
pkgSearchResult.configOpened = false;
return;
}
const pkg = pkgSearchResult.pkg;
const pkgName = pkg.name;
pkgSearchResult.configFetching = true;
pkgSearchResult.configOpened = true;
heliumService.getSinglePackageConfigs(pkg)
.then(confs => {
$scope.defaultPackageConfigs[pkgName] = confs;
pkgSearchResult.configFetching = false;
});
};
$scope.saveConfig = function(pkgSearchResult) {
const pkgName = pkgSearchResult.pkg.name;
const currentConf = $scope.defaultPackageConfigs[pkgName];
heliumService.saveConfig(pkgSearchResult.pkg, currentConf, () => {
// close after config is saved
pkgSearchResult.configOpened = false;
});
};
$scope.getDescriptionText = function(pkgSearchResult) {
return $sce.trustAsHtml(pkgSearchResult.pkg.description);
};
init();
}

50
zeppelin-web/src/app/helium/helium.css
View file

@ -79,6 +79,10 @@
width: 500px;
}
.spellConfigButton {
background-color: #FEFEFE;
}
.heliumPackageList .heliumPackageDisabledArtifact {
color:gray;
}
@ -132,4 +136,50 @@
color: #636363;
}
.heliumLearnMore {
margin-top:10px;
}
.heliumLearnMore a {
cursor:pointer;
margin-right:10px;
text-decoration:none;
}
.heliumRepoBtn {
margin-right: 8px;
}
.heliumRepoBtn:hover, .heliumRepoBtn:focus {
margin-right: 8px;
outline: 0;
}
.heliumRepoBtnToggled {
color: #333;
background-color: #e6e6e6;
border-color:#adadad;
}
.localPkgInfo {
margin: 10px 12px 0 0;
font-size: 11px;
font-style: italic;
color: #aaaaaa;
text-align: right;
}
.heliumConfig {
margin-top: 30px;
margin-bottom: 10px;
}
.heliumConfigTable {
margin-top: 15px;
vertical-align:middle;
margin-bottom: 15px;
}
.heliumConfigValueText {
vertical-align: top;
}

139
zeppelin-web/src/app/helium/helium.html
View file

@ -18,6 +18,24 @@ limitations under the License.
<h3 class="new_h3">
Helium
</h3>
<div class="pull-right heliumLearnMore">
<a target="_blank"
class="helium-repo-btn"
ng-href="https://zeppelin.apache.org/helium_packages.html"
tooltip-placement="bottom"
tooltip="Learn more">
<i class="icon-question" ng-style="{color: 'black'}"></i>
</a>
<button tabindex="0" role="button"
ng-repeat="pkgTypes in allPackageTypes"
class="btn btn-default btn-sm heliumRepoBtn helium-popover"
ng-class="($parent.pkgListByType === pkgTypes) ? 'heliumRepoBtnToggled' : ''"
ng-click="$parent.pkgListByType = pkgTypes">
<i class="fa fa-cube"></i>
{{pkgTypes}}
</button>
<p class="localPkgInfo">* Local registry package's name is gray colored.</p>
</div>
</div>
</div>
<div ng-show="bundleOrder.length > 1"
@ -29,13 +47,13 @@ limitations under the License.
<div class="btn-group" data-ng-repeat="pkgName in bundleOrder"
as-sortable-item>
<div class="btn btn-default btn-sm"
ng-bind-html='defaultVersions[pkgName].pkg.icon'
ng-bind-html='defaultPackages[pkgName].pkg.icon'
as-sortable-item-handle>
</div>
</div>
<span class="saveLink"
ng-show="bundleOrderChanged"
ng-click="saveBundleOrder()">
ng-show="bundleOrderChanged"
ng-click="saveBundleOrder()">
save
</span>
</div>
@ -44,62 +62,127 @@ limitations under the License.
</div>
<div class="box width-full heliumPackageContainer">
<div class="row"
style="padding-bottom: 15px"
ng-if="getPackageSize(defaultPackages, pkgListByType) === 0">
<div class="col-md-12 gray40-message">
<em>Currently there is no available package to be listed</em>
</div>
</div>
<div class="row heliumPackageList"
ng-repeat="(pkgName, pkgInfo) in defaultVersions">
ng-repeat="pkgSearchResult in defaultPackages | toArray:false | orderBy: 'pkg.published':true"
ng-show="$parent.pkgListByType === pkgSearchResult.pkg.type">
<div class="col-md-12">
<div class="heliumPackageHead">
<div class="heliumPackageIcon"
ng-bind-html=pkgInfo.pkg.icon></div>
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
ng-bind-html=pkgSearchResult.pkg.icon></div>
<div class="heliumPackageIcon"
ng-if="pkgSearchResult.pkg.type === 'INTERPRETER'"
ng-bind-html=intpDefaultIcon></div>
<div class="heliumPackageName">
<span ng-if="hasNpmLink(pkgInfo)">
<a target="_blank" href="https://www.npmjs.com/package/{{pkgName}}">{{pkgName}}</a>
<span ng-if="hasNpmLink(pkgSearchResult)">
<a target="_blank" href="https://www.npmjs.com/package/{{pkgSearchResult.pkg.name}}">{{pkgSearchResult.pkg.name}}</a>
</span>
<span ng-if="!hasNpmLink(pkgInfo)" ng-class="{'heliumLocalPackage': isLocalPackage(pkgInfo)}">
{{pkgName}}
<span ng-if="!hasNpmLink(pkgSearchResult) && !hasMavenLink(pkgSearchResult)" ng-class="{'heliumLocalPackage': isLocalPackage(pkgSearchResult)}">
{{pkgSearchResult.pkg.name}}
</span>
<span class="heliumType">{{pkgInfo.pkg.type}}</span>
<span ng-if="hasMavenLink(pkgSearchResult)">
<a target="_blank"
href="http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22{{pkgSearchResult.pkg.artifact.split('@')[0]}}%22%20AND%20v%3A%22{{pkgSearchResult.pkg.artifact.split('@')[1]}}%22">
{{pkgSearchResult.pkg.name}}
</a>
</span>
<span class="heliumType">{{pkgSearchResult.pkg.type}}</span>
</div>
<div ng-show="!pkgInfo.enabled"
ng-click="enable(pkgName, pkgInfo.pkg.artifact)"
<div ng-show="!pkgSearchResult.enabled"
ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId)"
class="btn btn-success btn-xs"
style="float:right">Enable</div>
<div ng-show="pkgInfo.enabled"
ng-click="disable(pkgName)"
<div ng-show="pkgSearchResult.enabled"
ng-click="disable(pkgSearchResult.pkg.name)"
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
class="btn btn-info btn-xs"
style="float:right">Disable</div>
<div ng-show="configExists(pkgSearchResult)"
ng-click="toggleConfigButton(pkgSearchResult)"
ng-class="getConfigButtonClass(pkgSearchResult)"
class="btn btn-default btn-xs spellConfigButton"
style="float:right; margin-right:5px;">Config</div>
</div>
<div ng-class="{heliumPackageDisabledArtifact: !pkgInfo.enabled, heliumPackageEnabledArtifact: pkgInfo.enabled}">
{{pkgInfo.pkg.artifact}}
<span ng-show="packageInfos[pkgName].length > 0"
ng-click="toggleVersions(pkgName)">
<div ng-class="{heliumPackageDisabledArtifact: !pkgSearchResult.enabled, heliumPackageEnabledArtifact: pkgSearchResult.enabled}">
{{pkgSearchResult.pkg.artifact}}
<span ng-show="pkgSearchResults[pkgSearchResult.pkg.name].length > 0"
ng-click="toggleVersions(pkgSearchResult.pkg.name)">
versions
</span>
</div>
<ul class="heliumPackageVersions"
ng-show="showVersions[pkgName]">
ng-show="showVersions[pkgSearchResult.pkg.name]">
<li class="heliumPackageDisabledArtifact"
ng-repeat="pkg in packageInfos[pkgName]">
{{pkg.pkg.artifact}} -
<span ng-click="enable(pkgName, pkg.pkg.artifact)"
ng-repeat="pkgSearchResult in pkgSearchResults[pkgSearchResult.pkg.name]">
{{pkgSearchResult.pkg.artifact}} -
<span ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId)"
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
style="margin-left:3px;cursor:pointer;text-decoration: underline;color:#3071a9">
enable
</span>
<a target="_blank"
ng-if="pkgSearchResult.pkg.type === 'INTERPRETER'"
style="margin-left:3px;cursor:pointer;text-decoration: underline;color:#3071a9"
href="http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22{{pkgSearchResult.pkg.artifact.split('@')[0]}}%22%20AND%20v%3A%22{{pkgSearchResult.pkg.artifact.split('@')[1]}}%22">
see more
</a>
</li>
</ul>
<div class="heliumPackageDescription">
{{pkgInfo.pkg.description}}
</div>
<div ng-if="pkgInfo.pkg.type === 'SPELL' && pkgInfo.pkg.spell"
<div class="heliumPackageDescription" ng-bind-html="getDescriptionText(pkgSearchResult)" />
<div ng-if="pkgSearchResult.pkg.type === 'SPELL' && pkgSearchResult.pkg.spell"
class="spellInfo">
<div>
<span class="spellInfoDesc">MAGIC</span>
<span class="spellInfoValue">{{pkgInfo.pkg.spell.magic}} </span>
<span class="spellInfoValue">{{pkgSearchResult.pkg.spell.magic}} </span>
</div>
<div>
<span class="spellInfoDesc">USAGE</span>
<pre class="spellUsage">{{pkgInfo.pkg.spell.usage}} </pre>
<pre class="spellUsage">{{pkgSearchResult.pkg.spell.usage}} </pre>
</div>
</div>
<!--start: config-->
<div class="heliumConfig" ng-if="configOpened(pkgSearchResult)">
<h5>Configuration</h5>
<table class="heliumConfigTable table table-striped">
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Value</th>
</tr>
<tr>
</tr>
<tr data-ng-repeat="cfg in defaultPackageConfigs[pkgSearchResult.pkg.name]">
<td style="vertical-align: middle;">{{cfg.name}}</td>
<td style="vertical-align: middle;">{{cfg.type}}</td>
<td style="vertical-align: middle;">{{cfg.description}}</td>
<td>
<div class="input-group">
<input type="text" class="form-control" style="border-radius: 5px;"
data-ng-model="cfg.value" placeholder="{{cfg.defaultValue}}" />
</div>
</td>
</tr>
</table>
<div>
<button class="btn btn-primary"
ng-click="saveConfig(pkgSearchResult)">Save</button>
<button class="btn btn-default"
ng-click="toggleConfigButton(pkgSearchResult)">Close</button>
</div>
</div>
<!--end: config-->
</div>
</div>
</div>

19
zeppelin-web/src/app/helium/index.js Normal file
View file

@ -0,0 +1,19 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import HeliumController from './helium.controller';
angular.module('zeppelinWebApp')
.controller('HeliumCtrl', HeliumController);

1
zeppelin-web/src/app/home/home.css
View file

@ -28,6 +28,7 @@ body {
}
.bodyAsIframe {
padding-top: 20px;
background: white;
}

2
zeppelin-web/src/app/home/home.html
View file

@ -82,7 +82,7 @@ limitations under the License.
ng-controller="ParagraphCtrl"
ng-init="init(currentParagraph, home.note)"
ng-class="columnWidthClass(currentParagraph.config.colWidth)"
class="paragraph-col">
style="margin: 0; padding: 0;">
<div id="{{currentParagraph.id}}_paragraphColumn"
ng-if="currentParagraph.results"
ng-include src="'app/notebook/paragraph/paragraph.html'"

4
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -12,6 +12,8 @@
* limitations under the License.
*/
import { ParagraphStatus, } from '../notebook/paragraph/paragraph.status';
angular.module('zeppelinWebApp').controller('InterpreterCtrl', InterpreterCtrl);
function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeout, $route) {
@ -114,7 +116,7 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou
isDownloading = true;
}
if (setting.status === 'ERROR' || setting.errorReason) {
if (setting.status === ParagraphStatus.ERROR || setting.errorReason) {
ngToast.danger({content: 'Error setting properties for interpreter \'' +
setting.group + '.' + setting.name + '\': ' + setting.errorReason,
verticalPosition: 'top', dismissOnTimeout: false});

4
zeppelin-web/src/app/jobmanager/jobs/job.controller.js
View file

@ -12,6 +12,8 @@
* limitations under the License.
*/
import { ParagraphStatus, } from '../../notebook/paragraph/paragraph.status';
angular.module('zeppelinWebApp').controller('JobCtrl', JobCtrl);
function JobCtrl($scope, $http, baseUrlSrv) {
@ -24,7 +26,7 @@ function JobCtrl($scope, $http, baseUrlSrv) {
$scope.getProgress = function() {
var statusList = _.pluck($scope.notebookJob.paragraphs, 'status');
var runningJob = _.countBy(statusList, function(status) {
if (status === 'FINISHED' || status === 'RUNNING') {
if (status === ParagraphStatus.RUNNING || status === ParagraphStatus.FINISHED) {
return 'matchCount';
} else {
return 'none';

2
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -74,6 +74,7 @@ limitations under the License.
<button type="button"
class="btn btn-primary btn-xs"
ng-class="isNoteRunning() ? 'disabled' : ''"
ng-if="ticket.principal && ticket.principal !== 'anonymous'"
ng-hide="viewOnly || note.config.personalizedMode !== 'true'"
ng-click="toggleNotePersonalizedMode()"
@ -83,6 +84,7 @@ limitations under the License.
</button>
<button type="button"
class="btn btn-default btn-xs"
ng-class="isNoteRunning() ? 'disabled' : ''"
ng-if="ticket.principal && ticket.principal !== 'anonymous'"
ng-hide="viewOnly || note.config.personalizedMode === 'true'"
ng-click="toggleNotePersonalizedMode()"

34
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -12,6 +12,8 @@
* limitations under the License.
*/
import { isParagraphRunning, } from './paragraph/paragraph.status';
angular.module('zeppelinWebApp').controller('NotebookCtrl', NotebookCtrl);
function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
@ -342,16 +344,19 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
$scope.$broadcast('closeTable');
};
/**
* @returns {boolean} true if one more paragraphs are running. otherwise return false.
*/
$scope.isNoteRunning = function() {
var running = false;
if (!$scope.note) { return false; }
for (var i = 0; i < $scope.note.paragraphs.length; i++) {
if ($scope.note.paragraphs[i].status === 'PENDING' || $scope.note.paragraphs[i].status === 'RUNNING') {
running = true;
break;
for (let i = 0; i < $scope.note.paragraphs.length; i++) {
if (isParagraphRunning($scope.note.paragraphs[i])) {
return true;
}
}
return running;
return false;
};
$scope.killSaveTimer = function() {
@ -422,7 +427,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
const trimmedNewName = newName.trim();
if (trimmedNewName.length > 0 && $scope.note.name !== trimmedNewName) {
$scope.note.name = trimmedNewName;
websocketMsgSrv.updateNote($scope.note.id, $scope.note.name, $scope.note.config);
websocketMsgSrv.renameNote($scope.note.id, $scope.note.name);
}
};
@ -709,6 +714,7 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
$scope.permissions.owners = angular.element('#selectOwners').val();
$scope.permissions.readers = angular.element('#selectReaders').val();
$scope.permissions.writers = angular.element('#selectWriters').val();
angular.element('.permissionsForm select').find('option:not([is-select2="false"])').remove();
}
$scope.restartInterpreter = function(interpeter) {
@ -979,15 +985,20 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
$location.path('/');
}
$scope.note = note;
$scope.paragraphUrl = $routeParams.paragraphId;
$scope.asIframe = $routeParams.asIframe;
if ($scope.paragraphUrl) {
note = cleanParagraphExcept($scope.paragraphUrl, note);
$scope.note = cleanParagraphExcept($scope.paragraphUrl, $scope.note);
$scope.$broadcast('$unBindKeyEvent', $scope.$unBindKeyEvent);
$rootScope.$broadcast('setIframe', $scope.asIframe);
initializeLookAndFeel();
return;
}
$scope.note = note;
initializeLookAndFeel();
//open interpreter binding setting when there're none selected
getInterpreterBindings();
getPermissions();
@ -1005,6 +1016,11 @@ function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope,
document.removeEventListener('keydown', $scope.keyboardShortcut);
});
$scope.$on('$unBindKeyEvent', function() {
document.removeEventListener('click', $scope.focusParagraphOnClick);
document.removeEventListener('keydown', $scope.keyboardShortcut);
});
angular.element(window).bind('resize', function() {
const actionbarHeight = document.getElementById('actionbar').lastElementChild.clientHeight;
angular.element(document.getElementById('content')).css('padding-top', actionbarHeight - 20);

15
zeppelin-web/src/app/notebook/notebook.controller.test.js
View file

@ -7,7 +7,8 @@ describe('Controller: NotebookCtrl', function() {
getNote: function() {},
listRevisionHistory: function() {},
getInterpreterBindings: function() {},
updateNote: function() {}
updateNote: function() {},
renameNote: function() {}
};
var baseUrlSrvMock = {
@ -100,24 +101,24 @@ describe('Controller: NotebookCtrl', function() {
});
it('should NOT update note name when updateNoteName() is called with an invalid name', function() {
spyOn(websocketMsgSrvMock, 'updateNote');
spyOn(websocketMsgSrvMock, 'renameNote');
scope.updateNoteName('');
expect(scope.note.name).toEqual(noteMock.name);
expect(websocketMsgSrvMock.updateNote).not.toHaveBeenCalled();
expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled();
scope.updateNoteName(' ');
expect(scope.note.name).toEqual(noteMock.name);
expect(websocketMsgSrvMock.updateNote).not.toHaveBeenCalled();
expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled();
scope.updateNoteName(scope.note.name);
expect(scope.note.name).toEqual(noteMock.name);
expect(websocketMsgSrvMock.updateNote).not.toHaveBeenCalled();
expect(websocketMsgSrvMock.renameNote).not.toHaveBeenCalled();
});
it('should update note name when updateNoteName() is called with a valid name', function() {
spyOn(websocketMsgSrvMock, 'updateNote');
spyOn(websocketMsgSrvMock, 'renameNote');
var newName = 'Your Note';
scope.updateNoteName(newName);
expect(scope.note.name).toEqual(newName);
expect(websocketMsgSrvMock.updateNote).toHaveBeenCalled();
expect(websocketMsgSrvMock.renameNote).toHaveBeenCalled();
});
it('should reload note info once per one "setNoteMenu" event', function() {

8
zeppelin-web/src/app/notebook/notebook.html
View file

@ -79,19 +79,19 @@ limitations under the License.
data-ng-model="permissions">
<p><span class="owners">Owners </span>
<select id="selectOwners" multiple="multiple">
<option ng-repeat="owner in permissions.owners" selected="selected">{{owner}}</option>
<option is-select2="false" ng-repeat="owner in permissions.owners" selected="selected">{{owner}}</option>
</select>
Owners can change permissions,read and write the note.
</p>
<p><span class="writers">Writers </span>
<select id="selectWriters" multiple="multiple">
<option ng-repeat="writers in permissions.writers" selected="selected">{{writers}}</option>
<option is-select2="false" ng-repeat="writers in permissions.writers" selected="selected">{{writers}}</option>
</select>
Writers can read and write the note.
</p>
<p><span class="readers">Readers </span>
<select id="selectReaders" multiple="multiple">
<option ng-repeat="readers in permissions.readers" selected="selected">{{readers}}</option>
<option is-select2="false" ng-repeat="readers in permissions.readers" selected="selected">{{readers}}</option>
</select>
Readers can only read the note.
</p>
@ -112,7 +112,7 @@ limitations under the License.
ng-controller="ParagraphCtrl"
ng-init="init(currentParagraph, note)"
ng-class="columnWidthClass(currentParagraph.config.colWidth)"
class="paragraph-col">
style="margin: 0; padding: 0;">
<div class="new-paragraph" ng-click="insertNew('above')" ng-hide="viewOnly || asIframe || revisionView">
<h4 class="plus-sign">&#43;</h4>
</div>

20
zeppelin-web/src/app/notebook/paragraph/paragraph-control.html
View file

@ -38,7 +38,7 @@ limitations under the License.
{{paragraph.status}}
</span>
<span ng-if="paragraph.status=='RUNNING'">
<span ng-if="paragraph.status === 'RUNNING' && paragraph.executor !== 'SPELL'">
{{getProgress()}}%
</span>
@ -89,6 +89,18 @@ limitations under the License.
</a>
</li>
<li role="separator" class="divider"></li>
<li style="padding-top:8px"
ng-if="paragraph.config.runOnSelectionChange == true || paragraph.config.runOnSelectionChange == false">
<span ng-if="paragraph.config.runOnSelectionChange == true" class="fa fa-toggle-on shortcut-icon" style="padding:3px 8px 8px 20px"></span>
<span ng-if="paragraph.config.runOnSelectionChange == false" class="fa fa-toggle-off shortcut-icon" style="padding:3px 8px 8px 20px"></span>Run on selection change
<form style="display:inline; float:right">
<input type="checkbox"
style="width:16px; margin-right:20px"
tooltip-placement="top" tooltip="Even if you uncheck this, still can run the paragraph by pressing Enter"
ng-checked="{{paragraph.config.runOnSelectionChange}}"
ng-click="turnOnAutoRun(paragraph)"/>
</form>
</li>
<li>
<a ng-click="$event.stopPropagation()" class="dropdown"><span class="fa fa-arrows-h shortcut-icon"></span>Width
<form style="display:inline; margin-left:5px; float:right">
@ -100,15 +112,15 @@ limitations under the License.
</a>
</li>
<li>
<a ng-click="moveUp(paragraph)" ng-hide="$first"><span class="icon-arrow-up shortcut-icon"></span>Move Up
<a ng-click="moveUp(paragraph)" ng-hide="$first"><span class="icon-arrow-up shortcut-icon"></span>Move up
<span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+k</span></a>
</li>
<li>
<a ng-click="moveDown(paragraph)" ng-hide="$last"><span class="icon-arrow-down shortcut-icon"></span>Move Down
<a ng-click="moveDown(paragraph)" ng-hide="$last"><span class="icon-arrow-down shortcut-icon"></span>Move down
<span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+j</span></a>
</li>
<li>
<a ng-click="insertNew('below')"><span class="icon-plus shortcut-icon"></span>Insert New
<a ng-click="insertNew('below')"><span class="icon-plus shortcut-icon"></span>Insert new
<span class="shortcut-keys">Ctrl+{{ isMac ? 'Option' : 'Alt'}}+b</span></a>
</li>
<li>

48
zeppelin-web/src/app/notebook/paragraph/paragraph-parameterizedQueryForm.html
View file

@ -19,33 +19,53 @@ limitations under the License.
ng-init="loadForm(formulaire, paragraph.settings.params)">
<label class="control-label input-sm" ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }">{{formulaire.name}}</label>
<div>
<input class="form-control input-sm"
ng-if="!paragraph.settings.forms[formulaire.name].options"
ng-enter="runParagraphFromButton(getEditorValue())"
ng-model="paragraph.settings.params[formulaire.name]"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
name="{{formulaire.name}}" />
</div>
<div ng-if="paragraph.config.runOnSelectionChange == true">
<select class="form-control input-sm"
ng-if="paragraph.settings.forms[formulaire.name].options && paragraph.settings.forms[formulaire.name].type != 'checkbox'"
ng-enter="runParagraphFromButton(getEditorValue())"
ng-change="runParagraphFromButton(getEditorValue())"
ng-model="paragraph.settings.params[formulaire.name]"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
name="{{formulaire.name}}"
ng-options="option.value as (option.displayName||option.value) for option in paragraph.settings.forms[formulaire.name].options">
</select>
<div ng-if="paragraph.settings.forms[formulaire.name].type == 'checkbox'">
<label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
class="checkbox-item input-sm">
<input type="checkbox"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
ng-click="toggleCheckbox(formulaire, option, false)"/>{{option.displayName||option.value}}
</label>
</div>
</div>
<div ng-if="paragraph.config.runOnSelectionChange == false">
<select class="form-control input-sm"
ng-if="paragraph.settings.forms[formulaire.name].options && paragraph.settings.forms[formulaire.name].type != 'checkbox'"
ng-enter="runParagraphFromButton(getEditorValue())"
ng-model="paragraph.settings.params[formulaire.name]"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
name="{{formulaire.name}}"
ng-options="option.value as (option.displayName||option.value) for option in paragraph.settings.forms[formulaire.name].options">
</select>
</div>
<div ng-if="paragraph.config.runOnSelectionChange == true &&
paragraph.settings.forms[formulaire.name].type == 'checkbox'">
<label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
class="checkbox-item input-sm">
<input type="checkbox"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
ng-click="toggleCheckbox(formulaire, option, false); runParagraphFromButton(getEditorValue())"/> {{option.displayName||option.value}}
</label>
</div>
<div ng-if="paragraph.config.runOnSelectionChange == false &&
paragraph.settings.forms[formulaire.name].type == 'checkbox'">
<label ng-repeat="option in paragraph.settings.forms[formulaire.name].options"
class="checkbox-item input-sm">
<input type="checkbox"
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }"
ng-checked="paragraph.settings.params[formulaire.name].indexOf(option.value) > -1"
ng-enter="runParagraphFromButton(getEditorValue())"
ng-click="toggleCheckbox(formulaire, option, false)"/> {{option.displayName||option.value}}
</label>
</div>
</div>
</form>

147
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -12,9 +12,16 @@
* limitations under the License.
*/
import { SpellResult, } from '../../spell';
import {
SpellResult,
} from '../../spell';
ParagraphStatus, isParagraphRunning,
} from './paragraph.status';
const ParagraphExecutor = {
SPELL: 'SPELL',
INTERPRETER: 'INTERPRETER',
NONE: '', /** meaning `DONE` */
};
angular.module('zeppelinWebApp').controller('ParagraphCtrl', ParagraphCtrl);
@ -25,6 +32,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
'ngInject';
var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_';
$rootScope.keys = Object.keys;
$scope.parentNote = null;
$scope.paragraph = {};
$scope.paragraph.results = {};
@ -32,6 +40,12 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
$scope.originalText = '';
$scope.editor = null;
// transactional info for spell execution
$scope.spellTransaction = {
totalResultCount: 0, renderedResultCount: 0,
propagated: false, resultsMsg: [], paragraphText: '',
};
var editorSetting = {};
// flag that is used to set editor setting on paste percent sign
var pastePercentSign = false;
@ -114,13 +128,25 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
};
var initializeDefault = function(config) {
var forms = $scope.paragraph.settings.forms;
if (!config.colWidth) {
config.colWidth = 12;
}
if (config.enabled === undefined) {
config.enabled = true;
}
for (var idx in forms) {
if (forms[idx]) {
if (forms[idx].options) {
if (config.runOnSelectionChange === undefined) {
config.runOnSelectionChange = true;
}
}
}
}
if (!config.results) {
config.results = {};
@ -202,7 +228,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
};
$scope.isRunning = function(paragraph) {
return paragraph.status === 'RUNNING' || paragraph.status === 'PENDING';
return isParagraphRunning(paragraph);
};
$scope.cancelParagraph = function(paragraph) {
@ -224,10 +250,9 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
$scope.handleSpellError = function(paragraphText, error,
digestRequired, propagated) {
const errorMessage = error.stack;
$scope.paragraph.status = 'ERROR';
$scope.paragraph.status = ParagraphStatus.ERROR;
$scope.paragraph.errorMessage = errorMessage;
console.error('Failed to execute interpret() in spell\n', error);
if (digestRequired) { $scope.$digest(); }
if (!propagated) {
$scope.propagateSpellResult(
@ -237,8 +262,50 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
}
};
$scope.runParagraphUsingSpell = function(spell, paragraphText,
$scope.prepareSpellTransaction = function(resultsMsg, propagated, paragraphText) {
$scope.spellTransaction.totalResultCount = resultsMsg.length;
$scope.spellTransaction.renderedResultCount = 0;
$scope.spellTransaction.propagated = propagated;
$scope.spellTransaction.resultsMsg = resultsMsg;
$scope.spellTransaction.paragraphText = paragraphText;
};
/**
* - update spell transaction count and
* - check transaction is finished based on the result count
* @returns {boolean}
*/
$scope.increaseSpellTransactionResultCount = function() {
$scope.spellTransaction.renderedResultCount += 1;
const total = $scope.spellTransaction.totalResultCount;
const current = $scope.spellTransaction.renderedResultCount;
return total === current;
};
$scope.cleanupSpellTransaction = function() {
const status = ParagraphStatus.FINISHED;
$scope.paragraph.executor = ParagraphExecutor.NONE;
$scope.paragraph.status = status;
$scope.paragraph.results.code = status;
const propagated = $scope.spellTransaction.propagated;
const resultsMsg = $scope.spellTransaction.resultsMsg;
const paragraphText = $scope.spellTransaction.paragraphText;
if (!propagated) {
const propagable = SpellResult.createPropagable(resultsMsg);
$scope.propagateSpellResult(
$scope.paragraph.id, $scope.paragraph.title,
paragraphText, propagable, status, '',
$scope.paragraph.config, $scope.paragraph.settings.params);
}
};
$scope.runParagraphUsingSpell = function(paragraphText,
magic, digestRequired, propagated) {
$scope.paragraph.status = 'RUNNING';
$scope.paragraph.executor = ParagraphExecutor.SPELL;
$scope.paragraph.results = {};
$scope.paragraph.errorMessage = '';
if (digestRequired) { $scope.$digest(); }
@ -248,30 +315,20 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
const splited = paragraphText.split(magic);
// remove leading spaces
const textWithoutMagic = splited[1].replace(/^\s+/g, '');
const spellResult = spell.interpret(textWithoutMagic);
const parsed = spellResult.getAllParsedDataWithTypes(
heliumService.getAllSpells(), magic, textWithoutMagic);
// handle actual result message in promise
parsed.then(resultsMsg => {
const status = 'FINISHED';
$scope.paragraph.status = status;
$scope.paragraph.results.code = status;
$scope.paragraph.results.msg = resultsMsg;
$scope.paragraph.config.tableHide = false;
if (digestRequired) { $scope.$digest(); }
heliumService.executeSpell(magic, textWithoutMagic)
.then(resultsMsg => {
$scope.prepareSpellTransaction(resultsMsg, propagated, paragraphText);
if (!propagated) {
const propagable = SpellResult.createPropagable(resultsMsg);
$scope.propagateSpellResult(
$scope.paragraph.id, $scope.paragraph.title,
paragraphText, propagable, status, '',
$scope.paragraph.config, $scope.paragraph.settings.params);
}
}).catch(error => {
$scope.handleSpellError(paragraphText, error,
digestRequired, propagated);
});
$scope.paragraph.results.msg = resultsMsg;
$scope.paragraph.config.tableHide = false;
})
.catch(error => {
$scope.handleSpellError(paragraphText, error,
digestRequired, propagated);
});
} catch (error) {
$scope.handleSpellError(paragraphText, error,
digestRequired, propagated);
@ -307,13 +364,10 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
if (!paragraphText || $scope.isRunning($scope.paragraph)) {
return;
}
const magic = SpellResult.extractMagic(paragraphText);
const spell = heliumService.getSpellByMagic(magic);
if (spell) {
$scope.runParagraphUsingSpell(
spell, paragraphText, magic, digestRequired, propagated);
if (heliumService.getSpellByMagic(magic)) {
$scope.runParagraphUsingSpell(paragraphText, magic, digestRequired, propagated);
} else {
$scope.runParagraphUsingBackendInterpreter(paragraphText);
}
@ -343,6 +397,11 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
$scope.runParagraph(paragraphText, false, false)
};
$scope.turnOnAutoRun = function (paragraph) {
paragraph.config.runOnSelectionChange = !paragraph.config.runOnSelectionChange;
commitParagraph(paragraph);
};
$scope.moveUp = function(paragraph) {
$scope.$emit('moveParagraphUp', paragraph);
};
@ -503,7 +562,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
if ($scope.asIframe) {
return 'col-md-12';
} else {
return 'col-md-' + n;
return 'paragraph-col col-md-' + n;
}
};
@ -1110,7 +1169,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
$scope.paragraph.title = newPara.title;
$scope.paragraph.lineNumbers = newPara.lineNumbers;
$scope.paragraph.status = newPara.status;
if (newPara.status !== 'RUNNING') {
if (newPara.status !== ParagraphStatus.RUNNING) {
$scope.paragraph.results = newPara.results;
}
$scope.paragraph.settings = newPara.settings;
@ -1134,7 +1193,8 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
const statusChanged = (newPara.status !== oldPara.status);
const resultRefreshed = (newPara.dateFinished !== oldPara.dateFinished) ||
isEmpty(newPara.results) !== isEmpty(oldPara.results) ||
newPara.status === 'ERROR' || (newPara.status === 'FINISHED' && statusChanged);
newPara.status === ParagraphStatus.ERROR ||
(newPara.status === ParagraphStatus.FINISHED && statusChanged);
// 2. update texts managed by $scope
$scope.updateAllScopeTexts(oldPara, newPara);
@ -1157,6 +1217,8 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
}
};
/** $scope.$on */
$scope.$on('runParagraphUsingSpell', function(event, data) {
const oldPara = $scope.paragraph;
let newPara = data.paragraph;
@ -1341,4 +1403,17 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
$scope.$on('closeTable', function(event) {
$scope.closeTable($scope.paragraph);
});
$scope.$on('resultRendered', function(event, paragraphId) {
if ($scope.paragraph.id !== paragraphId) {
return;
}
/** increase spell result count and return if not finished */
if (!$scope.increaseSpellTransactionResultCount()) {
return;
}
$scope.cleanupSpellTransaction();
});
}

5
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.test.js
View file

@ -5,7 +5,10 @@ describe('Controller: ParagraphCtrl', function() {
var scope;
var websocketMsgSrvMock = {};
var paragraphMock = {
config: {}
config: {},
settings: {
forms: {}
}
};
var route = {
current: {

12
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -118,7 +118,7 @@ table.dataTable.table-condensed .sorting_desc:after {
.paragraphAsIframe {
padding: 0;
margin-top: -79px;
margin-left: -10px;
margin-left: 0px;
margin-right: -10px;
}
@ -150,7 +150,7 @@ table.dataTable.table-condensed .sorting_desc:after {
display: block;
unicode-bidi: embed;
display: block !important;
margin: 0 0 10px!important;
margin: 0 10px 5px!important;
font-size: 12px!important;
line-height: 1.42857143!important;
word-break: break-all!important;
@ -158,6 +158,14 @@ table.dataTable.table-condensed .sorting_desc:after {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Consolas', 'source-code-pro', monospace;
}
.paragraphAsIframe .title {
width: 80%;
font-weight: bold;
font-family: 'Roboto', sans-serif;
font-size: 17px !important;
margin: 0 10px !important;
}
/*
Paragraph Controls CSS
*/

30
zeppelin-web/src/app/notebook/paragraph/paragraph.status.js Normal file
View file

@ -0,0 +1,30 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const ParagraphStatus = {
READY: 'READY',
PENDING: 'PENDING',
RUNNING: 'RUNNING',
FINISHED: 'FINISHED',
ABORT: 'ABORT',
ERROR: 'ERROR',
};
export function isParagraphRunning(paragraph) {
if (!paragraph) { return false; }
const status = paragraph.status;
if (!status) { return false; }
return status === ParagraphStatus.PENDING || status === ParagraphStatus.RUNNING;
}

210
zeppelin-web/src/app/notebook/paragraph/result/result.controller.js
View file

@ -23,6 +23,7 @@ import {
DefaultDisplayType,
SpellResult,
} from '../../../spell'
import { ParagraphStatus, } from '../paragraph.status';
angular.module('zeppelinWebApp').controller('ResultCtrl', ResultCtrl);
@ -198,7 +199,7 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
*/
if (paragraph.id === data.paragraphId &&
resultIndex === data.index &&
(paragraph.status === 'RUNNING' || paragraph.status === 'PENDING')) {
(paragraph.status === ParagraphStatus.PENDING || paragraph.status === ParagraphStatus.RUNNING)) {
if (DefaultDisplayType.TEXT !== $scope.type) {
$scope.type = DefaultDisplayType.TEXT;
@ -266,21 +267,27 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
};
$scope.renderDefaultDisplay = function(targetElemId, type, data, refresh) {
if (type === DefaultDisplayType.TABLE) {
$timeout(function() {
$scope.renderGraph(targetElemId, $scope.graphMode, refresh);
}, 10);
} else if (type === DefaultDisplayType.HTML) {
renderHtml(targetElemId, data);
} else if (type === DefaultDisplayType.ANGULAR) {
renderAngular(targetElemId, data);
} else if (type === DefaultDisplayType.TEXT) {
renderText(targetElemId, data);
} else if (type === DefaultDisplayType.ELEMENT) {
renderElem(targetElemId, data);
} else {
console.error(`Unknown Display Type: ${type}`);
const afterLoaded = () => {
if (type === DefaultDisplayType.TABLE) {
renderGraph(targetElemId, $scope.graphMode, refresh);
} else if (type === DefaultDisplayType.HTML) {
renderHtml(targetElemId, data);
} else if (type === DefaultDisplayType.ANGULAR) {
renderAngular(targetElemId, data);
} else if (type === DefaultDisplayType.TEXT) {
renderText(targetElemId, data);
} else if (type === DefaultDisplayType.ELEMENT) {
renderElem(targetElemId, data);
} else {
console.error(`Unknown Display Type: ${type}`);
}
}
retryUntilElemIsLoaded(targetElemId, afterLoaded);
// send message to parent that this result is rendered
const paragraphId = $scope.$parent.paragraph.id;
$scope.$emit('resultRendered', paragraphId);
};
const renderResult = function(type, refresh) {
@ -296,12 +303,7 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
renderApp(`p${appState.id}`, appState);
} else {
if (!DefaultDisplayType[type]) {
const spell = heliumService.getSpellByMagic(type);
if (!spell) {
console.error(`Can't execute spell due to unknown display type: ${type}`);
return;
}
$scope.renderCustomDisplay(type, data, spell);
$scope.renderCustomDisplay(type, data);
} else {
const targetElemId = $scope.createDisplayDOMId(`p${$scope.id}`, type);
$scope.renderDefaultDisplay(targetElemId, type, data, refresh);
@ -316,38 +318,40 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
/**
* Render multiple sub results for custom display
*/
$scope.renderCustomDisplay = function(type, data, spell) {
$scope.renderCustomDisplay = function(type, data) {
// get result from intp
const spellResult = spell.interpret(data.trim());
const parsed = spellResult.getAllParsedDataWithTypes(
heliumService.getAllSpells());
if (!heliumService.getSpellByMagic(type)) {
console.error(`Can't execute spell due to unknown display type: ${type}`);
return;
}
// custom display result can include multiple subset results
parsed.then(dataWithTypes => {
const containerDOMId = `p${$scope.id}_custom`;
const afterLoaded = () => {
const containerDOM = angular.element(`#${containerDOMId}`);
// Spell.interpret() can create multiple outputs
for(let i = 0; i < dataWithTypes.length; i++) {
const dt = dataWithTypes[i];
const data = dt.data;
const type = dt.type;
heliumService.executeSpellAsDisplaySystem(type, data)
.then(dataWithTypes => {
const containerDOMId = `p${$scope.id}_custom`;
const afterLoaded = () => {
const containerDOM = angular.element(`#${containerDOMId}`);
// Spell.interpret() can create multiple outputs
for(let i = 0; i < dataWithTypes.length; i++) {
const dt = dataWithTypes[i];
const data = dt.data;
const type = dt.type;
// prepare each DOM to be filled
const subResultDOMId = $scope.createDisplayDOMId(`p${$scope.id}_custom_${i}`, type);
const subResultDOM = document.createElement('div');
containerDOM.append(subResultDOM);
subResultDOM.setAttribute('id', subResultDOMId);
// prepare each DOM to be filled
const subResultDOMId = $scope.createDisplayDOMId(`p${$scope.id}_custom_${i}`, type);
const subResultDOM = document.createElement('div');
containerDOM.append(subResultDOM);
subResultDOM.setAttribute('id', subResultDOMId);
$scope.renderDefaultDisplay(subResultDOMId, type, data, true);
}
};
$scope.renderDefaultDisplay(subResultDOMId, type, data, true);
}
};
retryUntilElemIsLoaded(containerDOMId, afterLoaded);
}).catch(error => {
console.error(`Failed to render custom display: ${$scope.type}\n` + error);
});
retryUntilElemIsLoaded(containerDOMId, afterLoaded);
})
.catch(error => {
console.error(`Failed to render custom display: ${$scope.type}\n` + error);
});
};
/**
@ -378,50 +382,38 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
};
const renderElem = function(targetElemId, data) {
const afterLoaded = () => {
const elem = angular.element(`#${targetElemId}`);
handleData(() => { data(targetElemId) }, DefaultDisplayType.ELEMENT,
() => {}, /** HTML element will be filled with data. thus pass empty success callback */
(error) => { elem.html(`${error.stack}`); }
);
};
retryUntilElemIsLoaded(targetElemId, afterLoaded);
const elem = angular.element(`#${targetElemId}`);
handleData(() => { data(targetElemId) }, DefaultDisplayType.ELEMENT,
() => {}, /** HTML element will be filled with data. thus pass empty success callback */
(error) => { elem.html(`${error.stack}`); }
);
};
const renderHtml = function(targetElemId, data) {
const afterLoaded = () => {
const elem = angular.element(`#${targetElemId}`);
handleData(data, DefaultDisplayType.HTML,
(generated) => {
elem.html(generated);
elem.find('pre code').each(function(i, e) {
hljs.highlightBlock(e);
});
/*eslint new-cap: [2, {"capIsNewExceptions": ["MathJax.Hub.Queue"]}]*/
MathJax.Hub.Queue(['Typeset', MathJax.Hub, elem[0]]);
},
(error) => { elem.html(`${error.stack}`); }
);
};
retryUntilElemIsLoaded(targetElemId, afterLoaded);
const elem = angular.element(`#${targetElemId}`);
handleData(data, DefaultDisplayType.HTML,
(generated) => {
elem.html(generated);
elem.find('pre code').each(function(i, e) {
hljs.highlightBlock(e);
});
/*eslint new-cap: [2, {"capIsNewExceptions": ["MathJax.Hub.Queue"]}]*/
MathJax.Hub.Queue(['Typeset', MathJax.Hub, elem[0]]);
},
(error) => { elem.html(`${error.stack}`); }
);
};
const renderAngular = function(targetElemId, data) {
const afterLoaded = () => {
const elem = angular.element(`#${targetElemId}`);
const paragraphScope = noteVarShareService.get(`${paragraph.id}_paragraphScope`);
handleData(data, DefaultDisplayType.ANGULAR,
(generated) => {
elem.html(generated);
$compile(elem.contents())(paragraphScope);
},
(error) => { elem.html(`${error.stack}`); }
);
};
retryUntilElemIsLoaded(targetElemId, afterLoaded);
const elem = angular.element(`#${targetElemId}`);
const paragraphScope = noteVarShareService.get(`${paragraph.id}_paragraphScope`);
handleData(data, DefaultDisplayType.ANGULAR,
(generated) => {
elem.html(generated);
$compile(elem.contents())(paragraphScope);
},
(error) => { elem.html(`${error.stack}`); }
);
};
const getTextResultElemId = function (resultId) {
@ -429,25 +421,21 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
};
const renderText = function(targetElemId, data) {
const afterLoaded = () => {
const elem = angular.element(`#${targetElemId}`);
handleData(data, DefaultDisplayType.TEXT,
(generated) => {
// clear all lines before render
removeChildrenDOM(targetElemId);
const elem = angular.element(`#${targetElemId}`);
handleData(data, DefaultDisplayType.TEXT,
(generated) => {
// clear all lines before render
removeChildrenDOM(targetElemId);
if (generated) {
const divDOM = angular.element('<div></div>').text(generated);
elem.append(divDOM);
}
if (generated) {
const divDOM = angular.element('<div></div>').text(generated);
elem.append(divDOM);
}
elem.bind('mousewheel', (e) => { $scope.keepScrollDown = false; });
},
(error) => { elem.html(`${error.stack}`); }
);
};
retryUntilElemIsLoaded(targetElemId, afterLoaded);
elem.bind('mousewheel', (e) => { $scope.keepScrollDown = false; });
},
(error) => { elem.html(`${error.stack}`); }
);
};
const removeChildrenDOM = function(targetElemId) {
@ -480,14 +468,13 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
}
}
$scope.renderGraph = function(graphElemId, graphMode, refresh) {
const renderGraph = function(graphElemId, graphMode, refresh) {
// set graph height
const height = $scope.config.graph.height;
const graphElem = angular.element(`#${graphElemId}`);
graphElem.height(height);
if (!graphMode) { graphMode = 'table'; }
const tableElemId = `p${$scope.id}_${graphMode}`;
const builtInViz = builtInVisualizations[graphMode];
if (!builtInViz) { return; }
@ -502,9 +489,11 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
}
}
let afterLoaded = function() { /** will be overwritten */ };
if (!builtInViz.instance) { // not instantiated yet
// render when targetEl is available
const afterLoaded = (loadedElem) => {
afterLoaded = function(loadedElem) {
try {
const transformationSettingTargetEl = angular.element('#trsetting' + $scope.id + '_' + graphMode);
const visualizationSettingTargetEl = angular.element('#trsetting' + $scope.id + '_' + graphMode);
@ -543,12 +532,11 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
}
};
retryUntilElemIsLoaded(tableElemId, afterLoaded);
} else if (refresh) {
// when graph options or data are changed
console.log('Refresh data %o', tableData);
const afterLoaded = (loadedElem) => {
afterLoaded = function(loadedElem) {
const transformationSettingTargetEl = angular.element('#trsetting' + $scope.id + '_' + graphMode);
const visualizationSettingTargetEl = angular.element('#trsetting' + $scope.id + '_' + graphMode);
const config = getVizConfig(graphMode);
@ -562,15 +550,15 @@ function ResultCtrl($scope, $rootScope, $route, $window, $routeParams, $location
builtInViz.instance.renderSetting(visualizationSettingTargetEl);
};
retryUntilElemIsLoaded(tableElemId, afterLoaded);
} else {
const afterLoaded = (loadedElem) => {
afterLoaded = function(loadedElem) {
loadedElem.height(height);
builtInViz.instance.activate();
};
retryUntilElemIsLoaded(tableElemId, afterLoaded);
}
const tableElemId = `p${$scope.id}_${graphMode}`;
retryUntilElemIsLoaded(tableElemId, afterLoaded);
};
$scope.switchViz = function(newMode) {

3
zeppelin-web/src/app/spell/spell-base.js
View file

@ -31,9 +31,10 @@ export class SpellBase {
* Consumes text and return `SpellResult`.
*
* @param paragraphText {string} which doesn't include magic
* @param config {Object}
* @return {SpellResult}
*/
interpret(paragraphText) {
interpret(paragraphText, config) {
throw new Error('SpellBase.interpret() should be overrided');
}

2
zeppelin-web/src/app/visualization/builtins/visualization-areachart.js
View file

@ -60,7 +60,7 @@ export default class AreachartVisualization extends Nvd3ChartVisualization {
configureChart(chart) {
var self = this;
chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
chart.yAxisTickFormat(function(d) {return self.yAxisTickFormat(d);});
chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d);});
chart.yAxis.axisLabelDistance(50);
chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)

11
zeppelin-web/src/app/visualization/builtins/visualization-linechart.js
View file

@ -70,7 +70,12 @@ export default class LinechartVisualization extends Nvd3ChartVisualization {
configureChart(chart) {
var self = this;
chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d, self.xLabels);});
chart.yAxis.tickFormat(function(d) {
if (d === undefined) {
return 'N/A';
}
return self.yAxisTickFormat(d, self.xLabels);
});
chart.yAxis.axisLabelDistance(50);
if (chart.useInteractiveGuideline) { // lineWithFocusChart hasn't got useInteractiveGuideline
chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)
@ -111,4 +116,8 @@ export default class LinechartVisualization extends Nvd3ChartVisualization {
}
};
};
defaultY() {
return undefined;
};
}

Some files were not shown because too many files have changed in this diff Show more