mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge branch 'master' into ZEPPELIN-2341/includeSparkRByDefault
This commit is contained in:
Ahyoung Ryu
GitHub
commit
f891fd4bb3
39 changed files with 1440 additions and 153 deletions
16
.travis.yml
16
.travis.yml
|
|
@ -37,7 +37,7 @@ addons:
|
|||
env:
|
||||
global:
|
||||
# Interpreters does not required by zeppelin-server integration tests
|
||||
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy'
|
||||
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
|
||||
|
||||
matrix:
|
||||
include:
|
||||
|
|
@ -51,7 +51,7 @@ matrix:
|
|||
|
||||
# Test selenium with spark module for 1.6.3
|
||||
- jdk: "oraclejdk7"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
|
||||
# Test interpreter modules
|
||||
- jdk: "oraclejdk7"
|
||||
|
|
@ -59,27 +59,27 @@ matrix:
|
|||
|
||||
# Test spark module for 2.1.0 with scala 2.11, livy
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 2.0.2 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.10
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 2
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 3
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
before_install:
|
||||
# check files included in commit range, clear bower_components if a bower.json file has changed.
|
||||
|
|
|
|||
|
|
@ -259,13 +259,13 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreters</name>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
|
||||
<description>Comma separated interpreter configurations. First interpreter become a default</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.group.order</name>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy</value>
|
||||
<description></description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -106,8 +106,8 @@ function make_binary_release() {
|
|||
|
||||
git_clone
|
||||
make_source_package
|
||||
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}"
|
||||
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
|
||||
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pscala-${SCALA_VERSION}"
|
||||
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
|
||||
|
||||
# remove non release files and dirs
|
||||
rm -rf "${WORKING_DIR}/zeppelin"
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
|
|||
DO_SNAPSHOT="yes"
|
||||
fi
|
||||
|
||||
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pr"
|
||||
PROJECT_OPTIONS="-pl !zeppelin-distribution"
|
||||
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
|
||||
NEXUS_PROFILE="153446d1ac37c4"
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/geode.html">Geode</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/groovy.html">Groovy</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hdfs.html">HDFS</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ If you're unsure about the options, use the same commands that creates official
|
|||
# update all pom.xml to use scala 2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
# build zeppelin with all interpreters and include latest version of Apache spark support for local mode.
|
||||
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pscala-2.11
|
||||
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Psparkr -Pr -Pscala-2.11
|
||||
```
|
||||
|
||||
####3. Done
|
||||
|
|
@ -145,10 +145,6 @@ Available profiles are
|
|||
enable YARN support for local mode
|
||||
> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead.
|
||||
|
||||
##### `-Ppyspark` (optional)
|
||||
|
||||
enable [PySpark](http://spark.apache.org/docs/latest/api/python/) support for local mode.
|
||||
|
||||
##### `-Pr` (optional)
|
||||
|
||||
enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration.
|
||||
|
|
@ -188,14 +184,14 @@ Here are some examples with several options:
|
|||
```bash
|
||||
# build with spark-2.1, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
|
||||
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Psparkr -Pscala-2.11 -DskipTests
|
||||
|
||||
# build with spark-2.0, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Psparkr -Pscala-2.11 -DskipTests
|
||||
|
||||
# build with spark-1.6, scala-2.10
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Psparkr -DskipTests
|
||||
|
||||
# spark-cassandra integration
|
||||
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests -DskipTests
|
||||
|
|
@ -328,10 +324,10 @@ mvn clean package -Pbuild-distr
|
|||
To build a distribution with specific profiles, run:
|
||||
|
||||
```sh
|
||||
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark
|
||||
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn
|
||||
```
|
||||
|
||||
The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.
|
||||
The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.
|
||||
|
||||
The archive is generated under _`zeppelin-distribution/target`_ directory
|
||||
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ This assumes you've already cloned the project either on the host machine in the
|
|||
|
||||
```
|
||||
cd /zeppelin
|
||||
mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests
|
||||
./bin/zeppelin-daemon.sh start
|
||||
```
|
||||
|
||||
|
|
|
|||
116
docs/interpreter/groovy.md
Normal file
116
docs/interpreter/groovy.md
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Apache Groovy Interpreter for Apache Zeppelin"
|
||||
description: "Apache Groovy is a powerful, optionally typed and dynamic language, with static-typing and static compilation capabilities, for the Java platform aimed at improving developer productivity thanks to a concise, familiar and easy to learn syntax."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Groovy Interpreter for Apache Zeppelin
|
||||
|
||||
|
||||
### Samples
|
||||
|
||||
```groovy
|
||||
%groovy
|
||||
//get a parameter defined as z.angularBind('ngSearchParam', value, 'paragraph_id')
|
||||
//g is a context object for groovy to avoid mix with z object
|
||||
def param = g.angular('ngSearchParam')
|
||||
//send request https://www.googleapis.com/customsearch/v1?q=ngSearchParam_value
|
||||
def r = HTTP.get(
|
||||
//assume you defined the groovy interpreter property
|
||||
// `search_baseurl`='https://www.googleapis.com/customsearch/v1'
|
||||
//in groovy object o.getProperty('A') == o.'A' == o.A == o['A']
|
||||
url : g.search_baseurl,
|
||||
query: [ q: param ],
|
||||
headers: [
|
||||
'Accept':'application/json',
|
||||
//'Authorization:' : g.getProperty('search_auth'),
|
||||
]
|
||||
)
|
||||
//check response code
|
||||
if( r.response.code==200 ) {
|
||||
g.html().with{
|
||||
//g.html() renders %angular to output and returns groovy.xml.MarkupBuilder
|
||||
h2("the response ${r.response.code}")
|
||||
span( r.response.body )
|
||||
h2("headers")
|
||||
pre( r.response.headers.join('\n') )
|
||||
}
|
||||
} else {
|
||||
//just to show that it's possible to use println with multiline groovy string to render output
|
||||
println("""%angular
|
||||
<script> alert ("code=${r.response.code} \n msg=${r.response.message}") </script>
|
||||
""")
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
```groovy
|
||||
%groovy
|
||||
|
||||
//renders a table with headers a, b, c and two rows
|
||||
g.table(
|
||||
[
|
||||
['a','b','c'],
|
||||
['a1','b1','c1'],
|
||||
['a2','b2','c2'],
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### the `g` object
|
||||
|
||||
* `g.angular(String name)`
|
||||
|
||||
Returns angular object by name. Look up notebook scope first and then global scope.
|
||||
|
||||
|
||||
* `g.angularBind(String name, Object value)`
|
||||
|
||||
Assign a new `value` into angular object `name`
|
||||
|
||||
|
||||
* `java.util.Properties g.getProperties()`
|
||||
|
||||
returns all properties defined for this interpreter
|
||||
|
||||
|
||||
* `String g.getProperty('PROPERTY_NAME')`
|
||||
```groovy
|
||||
g.PROPERTY_NAME
|
||||
g.'PROPERTY_NAME'
|
||||
g['PROPERTY_NAME']
|
||||
g.getProperties().getProperty('PROPERTY_NAME')
|
||||
```
|
||||
|
||||
All above the accessor to named property defined in groovy interpreter.
|
||||
In this case with name `PROPERTY_NAME`
|
||||
|
||||
|
||||
* `groovy.xml.MarkupBuilder g.html()`
|
||||
|
||||
Starts or continues rendering of `%angular` to output and returns [groovy.xml.MarkupBuilder](http://groovy-lang.org/processing-xml.html#_markupbuilder)
|
||||
MarkupBuilder is usefull to generate html (xml)
|
||||
|
||||
* `void g.table(obj)`
|
||||
|
||||
starts or continues rendering table rows.
|
||||
|
||||
obj: List(rows) of List(columns) where first line is a header
|
||||
|
||||
|
||||
|
||||
|
|
@ -140,6 +140,11 @@ You can also set other Spark properties which are not listed in the table. For a
|
|||
<td>true</td>
|
||||
<td>Import implicits, UDF collection, and sql if set true.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>zeppelin.spark.enableSupportedVersionCheck</td>
|
||||
<td>true</td>
|
||||
<td>Do not change - developer only setting, not for production use</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
|
||||
|
|
|
|||
|
|
@ -130,12 +130,11 @@ mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.3 -Pscala-2.10
|
|||
-`-Pscala-2.10` tells maven to build with Scala v2.10.
|
||||
|
||||
|
||||
**Note:** You may wish to include additional build flags such as `-Ppyspark` or `-Psparkr`. See [the build section of github for more details](https://github.com/apache/zeppelin#build).
|
||||
|
||||
**Note:** You can build against any version of Spark that has a Zeppelin build profile available. The key is to make sure you check out the matching version of Spark to build. At the time of this writing, Spark 1.6 was the most recent Spark version available.
|
||||
|
||||
**Note:** On build failures. Having installed Zeppelin close to 30 times now, I will tell you that sometimes the build fails for seemingly no reason.
|
||||
As long as you didn't edit any code, it is unlikely the build is failing because of something you did. What does tend to happen, is some dependency that maven is trying to download is unreachable. If your build fails on this step here are some tips:
|
||||
|
||||
- Don't get discouraged.
|
||||
- Scroll up and read through the logs. There will be clues there.
|
||||
- Retry (that is, run the `mvn clean package -DskipTests -Pspark-1.6` again)
|
||||
|
|
@ -154,7 +153,7 @@ Use `ifconfig` to determine the host machine's IP address. If you are not famili
|
|||
|
||||
Open a web-browser on a machine connected to the same network as the host (or in the host operating system if using a virtual machine). Navigate to http://`yourip`:8080, where yourip is the IP address you found in `ifconfig`.
|
||||
|
||||
See the [Zeppelin tutorial](../tutorial/tutorial.md) for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.
|
||||
See the [Zeppelin tutorial](../tutorial/tutorial.html) for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.
|
||||
|
||||
##### Flink Test
|
||||
Create a new notebook named "Flink Test" and copy and paste the following code.
|
||||
|
|
@ -417,6 +416,6 @@ You should be able check the Flink and Spark webuis (at something like http://`y
|
|||
|
||||
### Next Steps
|
||||
|
||||
Check out the [tutorial](./tutorial.md) for more cool things you can do with your new toy!
|
||||
Check out the [tutorial](./tutorial.html) for more cool things you can do with your new toy!
|
||||
|
||||
[Join the community](http://zeppelin.apache.org/community.html), ask questions and contribute! Every little bit helps.
|
||||
|
|
|
|||
4
groovy/README.md
Normal file
4
groovy/README.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
## Groovy Interpreter
|
||||
|
||||
|
||||
[see groovy documentation](../docs/interpreter/groovy.md)
|
||||
166
groovy/pom-groovy-only.xml
Normal file
166
groovy/pom-groovy-only.xml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<!--version>0.6.2</version-->
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<relativePath>..</relativePath>
|
||||
</parent>
|
||||
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-groovy</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<name>Zeppelin: Groovy interpreter</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.codehaus.groovy/groovy-all -->
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>2.4.7</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<compilerArgs>
|
||||
<!--arg>-verbose</arg-->
|
||||
<arg>-Xlint:unchecked</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
<executions>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>${project.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>${project.packaging}</type>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!--this one only for independent groovy interpreter assembly-->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.5.3</version>
|
||||
<configuration>
|
||||
<descriptor>src/assembly/dep.xml</descriptor>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-archive</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
149
groovy/pom.xml
Normal file
149
groovy/pom.xml
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<!--version>0.6.2</version-->
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<relativePath>..</relativePath>
|
||||
</parent>
|
||||
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-groovy</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<name>Zeppelin: Groovy interpreter</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.codehaus.groovy/groovy-all -->
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>2.4.7</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<compilerArgs>
|
||||
<!--arg>-verbose</arg-->
|
||||
<arg>-Xlint:unchecked</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
<executions>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>${project.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>${project.packaging}</type>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
66
groovy/src/assembly/dep.xml
Normal file
66
groovy/src/assembly/dep.xml
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
|
||||
<id>bin</id>
|
||||
<baseDirectory>groovy</baseDirectory>
|
||||
<formats>
|
||||
<format>zip</format>
|
||||
</formats>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<filtered>true</filtered>
|
||||
<includes>
|
||||
<include>README*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.build.directory}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
<include>revision.txt</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<!--fileSet>
|
||||
<directory>${project.basedir}/src/main/groovy/classes/</directory>
|
||||
<outputDirectory>/classes/</outputDirectory>
|
||||
<includes>
|
||||
<include>*.groovy</include>
|
||||
</includes>
|
||||
</fileSet-->
|
||||
|
||||
<!--fileSet>
|
||||
<directory>${project.build.directory}/site</directory>
|
||||
<outputDirectory>docs</outputDirectory>
|
||||
</fileSet-->
|
||||
</fileSets>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<unpack>false</unpack>
|
||||
<scope>runtime</scope>
|
||||
<!--excludes>
|
||||
<exclude>junit:junit</exclude>
|
||||
</excludes-->
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
1
groovy/src/assembly/readme.txt
Normal file
1
groovy/src/assembly/readme.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
to assemble groovy interpreter separately
|
||||
369
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
Normal file
369
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.groovy;
|
||||
|
||||
|
||||
import java.io.StringWriter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
|
||||
import groovy.xml.MarkupBuilder;
|
||||
import groovy.lang.Closure;
|
||||
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
|
||||
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.AngularObject;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.display.Input.ParamOption;
|
||||
import org.apache.zeppelin.annotation.ZeppelinApi;
|
||||
import org.apache.zeppelin.interpreter.RemoteWorksController;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
|
||||
/**
|
||||
* Groovy interpreter for Zeppelin.
|
||||
*/
|
||||
public class GObject extends groovy.lang.GroovyObjectSupport {
|
||||
|
||||
Logger log;
|
||||
StringWriter out;
|
||||
Properties props;
|
||||
InterpreterContext interpreterContext;
|
||||
Map<String, Object> bindings;
|
||||
|
||||
|
||||
public GObject(Logger log, StringWriter out, Properties p, InterpreterContext ctx,
|
||||
Map<String, Object> bindings) {
|
||||
this.log = log;
|
||||
this.out = out;
|
||||
this.interpreterContext = ctx;
|
||||
this.props = p;
|
||||
this.bindings = bindings;
|
||||
}
|
||||
|
||||
public Object getProperty(String key) {
|
||||
if ("log".equals(key)) {
|
||||
return log;
|
||||
}
|
||||
return props.getProperty(key);
|
||||
}
|
||||
|
||||
public void setProperty(String key, Object value) {
|
||||
throw new RuntimeException("Set properties not supported: " + key + "=" + value);
|
||||
}
|
||||
|
||||
public Properties getProperties() {
|
||||
return props;
|
||||
}
|
||||
|
||||
private void startOutputType(String type) {
|
||||
StringBuffer sb = out.getBuffer();
|
||||
if (sb.length() > 0) {
|
||||
if (sb.length() < type.length() || !type.equals(sb.substring(0, type.length()))) {
|
||||
log.error("try to start output `" + type + "` after non-" + type + " started");
|
||||
}
|
||||
} else {
|
||||
out.append(type);
|
||||
out.append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* returns gui object
|
||||
*/
|
||||
public GUI getGui() {
|
||||
return interpreterContext.getGui();
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object input(String name) {
|
||||
return input(name, "");
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object input(String name, Object defaultValue) {
|
||||
return getGui().input(name, defaultValue);
|
||||
}
|
||||
|
||||
private ParamOption[] toParamOptions(Map<Object, String> options) {
|
||||
ParamOption[] paramOptions = new ParamOption[options.size()];
|
||||
int i = 0;
|
||||
for (Map.Entry<Object, String> e : options.entrySet()) {
|
||||
paramOptions[i++] = new ParamOption(e.getKey(), e.getValue());
|
||||
}
|
||||
return paramOptions;
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Map<Object, String> options) {
|
||||
return select(name, "", options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Object defaultValue, Map<Object, String> options) {
|
||||
return getGui().select(name, defaultValue, toParamOptions(options));
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Collection<Object> checkbox(String name, Map<Object, String> options) {
|
||||
return checkbox(name, options.keySet(), options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Collection<Object> checkbox(String name, Collection<Object> defaultChecked,
|
||||
Map<Object, String> options) {
|
||||
return getGui().checkbox(name, defaultChecked, toParamOptions(options));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns shared variable if it was previously set. The same as getting groovy script variables
|
||||
* but this method will return null if script variable not assigned. To understand groovy script
|
||||
* variables see groovy.transform.Field annotation for more information.
|
||||
*
|
||||
* @see #put
|
||||
*/
|
||||
public Object get(String varName) {
|
||||
return bindings.get(varName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns script (shared) variable value but if value was not set returns default value. The same
|
||||
* as getting groovy script variables but this method will return default value if script variable
|
||||
* not assigned. To understand groovy script variables see groovy.transform.Field annotation for
|
||||
* more information.
|
||||
*
|
||||
* @see #put
|
||||
*/
|
||||
public Object get(String varName, Object defValue) {
|
||||
return bindings.containsKey(varName) ? bindings.get(varName) : defValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a new value to interpreter's shared variables.
|
||||
* Could be set by <code>put('varName', newValue )</code>
|
||||
* or by just assigning <code>varName = value</code> without declaring a variable.
|
||||
*/
|
||||
public Object put(String varName, Object newValue) {
|
||||
return bindings.put(varName, newValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* starts or continues rendering html/angular and returns MarkupBuilder to build html.
|
||||
* <pre> g.html().with{
|
||||
* h1("hello")
|
||||
* h2("world")
|
||||
* }</pre>
|
||||
*/
|
||||
public MarkupBuilder html() {
|
||||
startOutputType("%angular");
|
||||
return new MarkupBuilder(out);
|
||||
}
|
||||
|
||||
/**
|
||||
* starts or continues rendering table rows
|
||||
*
|
||||
* @param obj: 1. List(rows) of List(columns) where first line is a header
|
||||
*/
|
||||
public void table(Object obj) {
|
||||
if (obj == null) {
|
||||
return;
|
||||
}
|
||||
StringBuffer sb = out.getBuffer();
|
||||
startOutputType("%table");
|
||||
if (obj instanceof groovy.lang.Closure) {
|
||||
//if closure run and get result collection
|
||||
obj = ((Closure) obj).call();
|
||||
}
|
||||
if (obj instanceof Collection) {
|
||||
int count = 0;
|
||||
for (Object row : ((Collection) obj)) {
|
||||
count++;
|
||||
boolean rowStarted = false;
|
||||
if (row instanceof Collection) {
|
||||
for (Object field : ((Collection) row)) {
|
||||
if (rowStarted) {
|
||||
sb.append('\t');
|
||||
}
|
||||
sb.append(field);
|
||||
rowStarted = true;
|
||||
}
|
||||
} else {
|
||||
sb.append(row);
|
||||
}
|
||||
sb.append('\n');
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("Not supported table value :" + obj.getClass());
|
||||
}
|
||||
}
|
||||
|
||||
private AngularObject getAngularObject(String name) {
|
||||
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
// try get local object
|
||||
AngularObject paragraphAo = registry.get(name, noteId, interpreterContext.getParagraphId());
|
||||
AngularObject noteAo = registry.get(name, noteId, null);
|
||||
|
||||
AngularObject ao = paragraphAo != null ? paragraphAo : noteAo;
|
||||
|
||||
if (ao == null) {
|
||||
// then global object
|
||||
ao = registry.get(name, null, null);
|
||||
}
|
||||
return ao;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get angular object. Look up notebook scope first and then global scope
|
||||
*
|
||||
* @param name variable name
|
||||
* @return value
|
||||
*/
|
||||
public Object angular(String name) {
|
||||
AngularObject ao = getAngularObject(name);
|
||||
if (ao == null) {
|
||||
return null;
|
||||
} else {
|
||||
return ao.get();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void angularBind(String name, Object o, String noteId) {
|
||||
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
|
||||
|
||||
if (registry.get(name, noteId, null) == null) {
|
||||
registry.add(name, o, noteId, null);
|
||||
} else {
|
||||
registry.get(name, noteId, null).set(o);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create angular variable in notebook scope and bind with front end Angular display system.
|
||||
* If variable exists, it'll be overwritten.
|
||||
*
|
||||
* @param name name of the variable
|
||||
* @param o value
|
||||
*/
|
||||
public void angularBind(String name, Object o) {
|
||||
angularBind(name, o, interpreterContext.getNoteId());
|
||||
}
|
||||
|
||||
/*------------------------------------------RUN----------------------------------------*/
|
||||
@ZeppelinApi
|
||||
public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
|
||||
String paragraphId, InterpreterContext interpreterContext) {
|
||||
RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
|
||||
if (remoteWorksController != null) {
|
||||
return remoteWorksController.getRemoteContextRunner(noteId, paragraphId);
|
||||
}
|
||||
return new LinkedList<InterpreterContextRunner>();
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
|
||||
InterpreterContext interpreterContext) {
|
||||
RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
|
||||
if (remoteWorksController != null) {
|
||||
return remoteWorksController.getRemoteContextRunner(noteId);
|
||||
}
|
||||
return new LinkedList<InterpreterContextRunner>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId) {
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String paragraphId) {
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId, InterpreterContext context) {
|
||||
if (paragraphId.equals(context.getParagraphId())) {
|
||||
throw new InterpreterException("Can not run current Paragraph");
|
||||
}
|
||||
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, paragraphId,
|
||||
context);
|
||||
if (runners.size() <= 0) {
|
||||
throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size());
|
||||
}
|
||||
for (InterpreterContextRunner r : runners) {
|
||||
r.run();
|
||||
}
|
||||
}
|
||||
|
||||
public void runNote(String noteId) {
|
||||
runNote(noteId, interpreterContext);
|
||||
}
|
||||
|
||||
public void runNote(String noteId, InterpreterContext context) {
|
||||
String runningNoteId = context.getNoteId();
|
||||
String runningParagraphId = context.getParagraphId();
|
||||
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
|
||||
|
||||
if (runners.size() <= 0) {
|
||||
throw new InterpreterException("Note " + noteId + " not found " + runners.size());
|
||||
}
|
||||
|
||||
for (InterpreterContextRunner r : runners) {
|
||||
if (r.getNoteId().equals(runningNoteId) && r.getParagraphId().equals(runningParagraphId)) {
|
||||
continue;
|
||||
}
|
||||
r.run();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all paragraphs. except this.
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void runAll() {
|
||||
runAll(interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all paragraphs. except this.
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void runAll(InterpreterContext context) {
|
||||
runNote(context.getNoteId());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.groovy;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.apache.zeppelin.scheduler.Job;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
import org.apache.zeppelin.scheduler.SchedulerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import groovy.lang.GroovyShell;
|
||||
import groovy.lang.Script;
|
||||
import org.codehaus.groovy.control.CompilerConfiguration;
|
||||
import org.codehaus.groovy.runtime.ResourceGroovyMethods;
|
||||
import org.codehaus.groovy.runtime.StackTraceUtils;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Groovy interpreter for Zeppelin.
|
||||
*/
|
||||
public class GroovyInterpreter extends Interpreter {
|
||||
|
||||
Logger log = LoggerFactory.getLogger(GroovyInterpreter.class);
|
||||
GroovyShell shell = null; //new GroovyShell();
|
||||
//here we will store Interpreters shared variables. concurrent just in case.
|
||||
Map<String, Object> sharedBindings = new ConcurrentHashMap<String, Object>();
|
||||
//cache for groovy compiled scripts
|
||||
Map<String, Class<Script>> scriptCache = Collections
|
||||
.synchronizedMap(new WeakHashMap<String, Class<Script>>(100));
|
||||
|
||||
|
||||
public GroovyInterpreter(Properties property) {
|
||||
super(property);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open() {
|
||||
CompilerConfiguration conf = new CompilerConfiguration();
|
||||
conf.setDebug(true);
|
||||
shell = new GroovyShell(conf);
|
||||
String classes = getProperty("GROOVY_CLASSES");
|
||||
if (classes == null || classes.length() == 0) {
|
||||
try {
|
||||
File jar = new File(
|
||||
GroovyInterpreter.class.getProtectionDomain().getCodeSource().getLocation().toURI()
|
||||
.getPath());
|
||||
classes = new File(jar.getParentFile(), "classes").toString();
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
log.info("groovy classes classpath: " + classes);
|
||||
if (classes != null && classes.length() > 0) {
|
||||
File fClasses = new File(classes);
|
||||
if (!fClasses.exists()) {
|
||||
fClasses.mkdirs();
|
||||
}
|
||||
shell.getClassLoader().addClasspath(classes);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
shell = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FormType getFormType() {
|
||||
return FormType.NATIVE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getProgress(InterpreterContext context) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scheduler getScheduler() {
|
||||
return SchedulerFactory.singleton()
|
||||
.createOrGetParallelScheduler(GroovyInterpreter.class.getName() + this.hashCode(), 10);
|
||||
}
|
||||
|
||||
private Job getRunningJob(String paragraphId) {
|
||||
Job foundJob = null;
|
||||
Collection<Job> jobsRunning = getScheduler().getJobsRunning();
|
||||
for (Job job : jobsRunning) {
|
||||
if (job.getId().equals(paragraphId)) {
|
||||
foundJob = job;
|
||||
}
|
||||
}
|
||||
return foundJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Script getGroovyScript(String id, String scriptText) /*throws SQLException*/ {
|
||||
if (shell == null) {
|
||||
throw new RuntimeException("Groovy Shell is not initialized: null");
|
||||
}
|
||||
try {
|
||||
Class<Script> clazz = scriptCache.get(scriptText);
|
||||
if (clazz == null) {
|
||||
String scriptName = id + "_" + Long.toHexString(scriptText.hashCode()) + ".groovy";
|
||||
clazz = (Class<Script>) shell.parse(scriptText, scriptName).getClass();
|
||||
scriptCache.put(scriptText, clazz);
|
||||
}
|
||||
|
||||
Script script = (Script) clazz.newInstance();
|
||||
return script;
|
||||
} catch (Throwable t) {
|
||||
throw new RuntimeException("Failed to parse groovy script: " + t, t);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> predefinedBindings = new HashSet<String>();
|
||||
|
||||
static {
|
||||
predefinedBindings.add("g");
|
||||
predefinedBindings.add("out");
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
|
||||
try {
|
||||
Script script = getGroovyScript(contextInterpreter.getParagraphId(), cmd);
|
||||
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
|
||||
runningJob.info()
|
||||
.put("CURRENT_THREAD", Thread.currentThread()); //to be able to terminate thread
|
||||
Map<String, Object> bindings = script.getBinding().getVariables();
|
||||
bindings.clear();
|
||||
StringWriter out = new StringWriter((int) (cmd.length() * 1.75));
|
||||
//put shared bindings evaluated in this interpreter
|
||||
bindings.putAll(sharedBindings);
|
||||
//put predefined bindings
|
||||
bindings.put("g", new GObject(log, out, this.getProperty(), contextInterpreter, bindings));
|
||||
bindings.put("out", new PrintWriter(out, true));
|
||||
|
||||
script.run();
|
||||
//let's get shared variables defined in current script and store them in shared map
|
||||
for (Map.Entry<String, Object> e : bindings.entrySet()) {
|
||||
if (!predefinedBindings.contains(e.getKey())) {
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("groovy script variable " + e); //let's see what we have...
|
||||
}
|
||||
sharedBindings.put(e.getKey(), e.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
bindings.clear();
|
||||
InterpreterResult result = new InterpreterResult(Code.SUCCESS, out.toString());
|
||||
return result;
|
||||
} catch (Throwable t) {
|
||||
t = StackTraceUtils.deepSanitize(t);
|
||||
String msg = t.toString() + "\n at " + t.getStackTrace()[0];
|
||||
log.error("Failed to run script: " + t + "\n" + cmd + "\n", t);
|
||||
return new InterpreterResult(Code.ERROR, msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
Job runningJob = getRunningJob(context.getParagraphId());
|
||||
if (runningJob != null) {
|
||||
Map<String, Object> info = runningJob.info();
|
||||
Object object = info.get("CURRENT_THREAD");
|
||||
if (object instanceof Thread) {
|
||||
try {
|
||||
Thread t = (Thread) object;
|
||||
t.dumpStack();
|
||||
t.interrupt();
|
||||
//t.stop(); //TODO: need some way to terminate maybe through GObject..
|
||||
} catch (Throwable t) {
|
||||
log.error("Failed to cancel script: " + t, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
154
groovy/src/main/resources/HTTP.groovy
Normal file
154
groovy/src/main/resources/HTTP.groovy
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
/**
|
||||
* simple http rest client for groovy
|
||||
* by dlukyanov@ukr.net
|
||||
*/
|
||||
@groovy.transform.CompileStatic
|
||||
public class HTTP{
|
||||
//default response handler
|
||||
public static Closure TEXT_RECEIVER = {InputStream instr,Map ctx->
|
||||
return instr.getText( (String)ctx.encoding );
|
||||
}
|
||||
|
||||
public static Closure JSON_RECEIVER = { InputStream instr, Map ctx->
|
||||
return new groovy.json.JsonSlurper().parse(instr,(String)ctx.encoding);
|
||||
}
|
||||
|
||||
public static Closure FILE_RECEIVER(File f){
|
||||
return { InputStream instr, Map ctx->
|
||||
f<<instr;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String,Object> get(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','GET');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> post(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','POST');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> put(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','PUT');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> delete(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','DELETE');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> send(Map<String,Object> ctx)throws IOException{
|
||||
String url = ctx.url;
|
||||
Map<String,String> headers = (Map<String,String>)ctx.headers;
|
||||
String method = ctx.method;
|
||||
Object body = ctx.body;
|
||||
String encoding = ctx.encoding?:"UTF-8";
|
||||
Closure receiver = (Closure)ctx.receiver;
|
||||
Map<String,String> query = (Map<String,String>)ctx.query;
|
||||
|
||||
//copy context and set default values
|
||||
ctx = [:] + ctx;
|
||||
ctx.encoding = encoding;
|
||||
String contentType="";
|
||||
|
||||
if(query){
|
||||
url+="?"+query.collect{k,v-> k+"="+URLEncoder.encode(v,'UTF-8') }.join('&')
|
||||
}
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
|
||||
connection.setDoOutput(true);
|
||||
connection.setRequestMethod(method);
|
||||
if ( headers!=null && !headers.isEmpty() ) {
|
||||
//add headers
|
||||
for (Map.Entry<String, String> entry : headers.entrySet()) {
|
||||
connection.addRequestProperty(entry.getKey(), entry.getValue());
|
||||
if("content-type".equals(entry.getKey().toLowerCase()))contentType=entry.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
if(body!=null){
|
||||
//write body
|
||||
OutputStream out = connection.getOutputStream();
|
||||
if( body instanceof Closure ){
|
||||
((Closure)body).call(out, ctx);
|
||||
}else if(body instanceof InputStream){
|
||||
out << (InputStream)body;
|
||||
}else if(body instanceof Map){
|
||||
if( contentType.matches("(?i)[^/]+/json") ){
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append( JsonOutput.toJson((Map)body) );
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Map body type supported only for */json content-type");
|
||||
}
|
||||
}else if(body instanceof CharSequence){
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append((CharSequence)body);
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Unsupported body type: "+body.getClass());
|
||||
}
|
||||
out.flush();
|
||||
out.close();
|
||||
out=null;
|
||||
}
|
||||
|
||||
Map response = [:];
|
||||
ctx.response = response;
|
||||
response.code = connection.getResponseCode();
|
||||
response.message = connection.getResponseMessage();
|
||||
response.headers = connection.getHeaderFields();
|
||||
|
||||
InputStream instr = null;
|
||||
|
||||
if( ((int)response.code)>=400 ){
|
||||
try{
|
||||
instr = connection.getErrorStream();
|
||||
}catch(Exception ei){}
|
||||
}else{
|
||||
try{
|
||||
instr = connection.getInputStream();
|
||||
}catch(java.io.IOException ei){
|
||||
throw new IOException("fail to open InputStream for http code "+response.code+":"+ei);
|
||||
}
|
||||
}
|
||||
|
||||
if(instr!=null) {
|
||||
instr = new BufferedInputStream(instr);
|
||||
if(receiver==null){
|
||||
if( response.headers['Content-Type']?.toString()?.indexOf('/json')>0 ){
|
||||
receiver=JSON_RECEIVER;
|
||||
} else receiver=TEXT_RECEIVER;
|
||||
}
|
||||
response.body = receiver(instr,ctx);
|
||||
instr.close();
|
||||
instr=null;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
}
|
||||
15
groovy/src/main/resources/interpreter-setting.json
Normal file
15
groovy/src/main/resources/interpreter-setting.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[
|
||||
{
|
||||
"group": "groovy",
|
||||
"name": "groovy",
|
||||
"className": "org.apache.zeppelin.groovy.GroovyInterpreter",
|
||||
"properties": {
|
||||
"GROOVY_CLASSES": {
|
||||
"envName": null,
|
||||
"propertyName": "GROOVY_CLASSES",
|
||||
"defaultValue": "",
|
||||
"description": "The path for custom groovy classes location. If empty `./interpreter/groovy/classes`"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
1
pom.xml
1
pom.xml
|
|
@ -56,6 +56,7 @@
|
|||
<module>zeppelin-zengine</module>
|
||||
<module>zeppelin-display</module>
|
||||
<module>spark-dependencies</module>
|
||||
<module>groovy</module>
|
||||
<module>spark</module>
|
||||
<module>markdown</module>
|
||||
<module>angular</module>
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
<name>Zeppelin: Python interpreter</name>
|
||||
|
||||
<properties>
|
||||
<py4j.version>0.9.2</py4j.version>
|
||||
<python.py4j.version>0.9.2</python.py4j.version>
|
||||
<python.test.exclude>
|
||||
**/PythonInterpreterWithPythonInstalledTest.java,
|
||||
**/PythonInterpreterPandasSqlTest.java,
|
||||
|
|
@ -58,7 +58,7 @@
|
|||
<dependency>
|
||||
<groupId>net.sf.py4j</groupId>
|
||||
<artifactId>py4j</artifactId>
|
||||
<version>${py4j.version}</version>
|
||||
<version>${python.py4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -108,8 +108,8 @@
|
|||
<goals><goal>download-single</goal></goals>
|
||||
<configuration>
|
||||
<url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
|
||||
<fromFile>py4j-${py4j.version}.zip</fromFile>
|
||||
<toFile>${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip</toFile>
|
||||
<fromFile>py4j-${python.py4j.version}.zip</fromFile>
|
||||
<toFile>${project.build.directory}/../../interpreter/python/py4j-${python.py4j.version}.zip</toFile>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -123,7 +123,7 @@
|
|||
<phase>package</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip"
|
||||
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${python.py4j.version}.zip"
|
||||
dest="${project.build.directory}/../../interpreter/python"/>
|
||||
</target>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ This assumes you've already cloned the project either on the host machine in the
|
|||
|
||||
```
|
||||
cd /zeppelin
|
||||
mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests
|
||||
./bin/zeppelin-daemon.sh start
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ echo 'mvn clean package -DskipTests'
|
|||
echo
|
||||
echo '# or for a specific Spark/Hadoop build with additional options such as python and R support'
|
||||
echo
|
||||
echo 'mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests'
|
||||
echo 'mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests'
|
||||
echo './bin/zeppelin-daemon.sh start'
|
||||
echo
|
||||
echo 'On your host machine browse to http://localhost:8080/'
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@
|
|||
<spark.bin.download.url>
|
||||
http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz
|
||||
</spark.bin.download.url>
|
||||
<py4j.version>0.8.2.1</py4j.version>
|
||||
<spark.py4j.version>0.8.2.1</spark.py4j.version>
|
||||
|
||||
<!--plugin versions-->
|
||||
<plugin.shade.version>2.3</plugin.shade.version>
|
||||
|
|
@ -514,7 +514,7 @@
|
|||
<id>spark-1.6</id>
|
||||
<properties>
|
||||
<spark.version>1.6.3</spark.version>
|
||||
<py4j.version>0.9</py4j.version>
|
||||
<spark.py4j.version>0.9</spark.py4j.version>
|
||||
<akka.group>com.typesafe.akka</akka.group>
|
||||
<akka.version>2.3.11</akka.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
|
|
@ -526,7 +526,7 @@
|
|||
<properties>
|
||||
<spark.version>2.0.2</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.3</py4j.version>
|
||||
<spark.py4j.version>0.10.3</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -539,7 +539,7 @@
|
|||
<properties>
|
||||
<spark.version>2.1.0</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.4</py4j.version>
|
||||
<spark.py4j.version>0.10.4</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -827,68 +827,6 @@
|
|||
</dependencies>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>pyspark</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-pyspark-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<unpack>true</unpack>
|
||||
<url>${spark.src.download.url}</url>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<configuration>
|
||||
<filesets>
|
||||
<fileset>
|
||||
<directory>${basedir}/../python/build</directory>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>zip-pyspark-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<delete dir="../interpreter/spark/pyspark"/>
|
||||
<copy todir="../interpreter/spark/pyspark"
|
||||
file="${project.build.directory}/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/>
|
||||
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
|
||||
basedir="${project.build.directory}/${spark.archive}/python"
|
||||
includes="pyspark/*.py,pyspark/**/*.py"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
|
|
@ -994,6 +932,63 @@
|
|||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- include pyspark by default -->
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-pyspark-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<unpack>true</unpack>
|
||||
<url>${spark.src.download.url}</url>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<configuration>
|
||||
<filesets>
|
||||
<fileset>
|
||||
<directory>${basedir}/../python/build</directory>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>zip-pyspark-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<delete dir="../interpreter/spark/pyspark"/>
|
||||
<copy todir="../interpreter/spark/pyspark"
|
||||
file="${project.build.directory}/${spark.archive}/python/lib/py4j-${spark.py4j.version}-src.zip"/>
|
||||
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
|
||||
basedir="${project.build.directory}/${spark.archive}/python"
|
||||
includes="pyspark/*.py,pyspark/**/*.py"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- include sparkr by default -->
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
|
|
|
|||
|
|
@ -537,7 +537,7 @@
|
|||
<id>spark-1.6</id>
|
||||
<properties>
|
||||
<spark.version>1.6.3</spark.version>
|
||||
<py4j.version>0.9</py4j.version>
|
||||
<spark.py4j.version>0.9</spark.py4j.version>
|
||||
<akka.group>com.typesafe.akka</akka.group>
|
||||
<akka.version>2.3.11</akka.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
|
|
@ -549,7 +549,7 @@
|
|||
<properties>
|
||||
<spark.version>2.0.2</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.3</py4j.version>
|
||||
<spark.py4j.version>0.10.3</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -562,7 +562,7 @@
|
|||
<properties>
|
||||
<spark.version>2.1.0</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.4</py4j.version>
|
||||
<spark.py4j.version>0.10.4</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
|
|||
|
|
@ -337,7 +337,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
public InterpreterResult interpret(String st, InterpreterContext context) {
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
sparkInterpreter.populateSparkWebUrl(context);
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion()) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
|
@ -466,8 +466,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
//start code for completion
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion() == false
|
||||
&& pythonscriptRunning == false) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion() || pythonscriptRunning == false) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import org.apache.spark.repl.SparkILoop;
|
|||
import org.apache.spark.scheduler.ActiveJob;
|
||||
import org.apache.spark.scheduler.DAGScheduler;
|
||||
import org.apache.spark.scheduler.Pool;
|
||||
import org.apache.spark.scheduler.SparkListenerApplicationEnd;
|
||||
import org.apache.spark.scheduler.SparkListenerJobStart;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.spark.ui.SparkUI;
|
||||
|
|
@ -128,7 +127,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
private static File outputDir; // class outputdir for scala 2.11
|
||||
private Object classServer; // classserver for scala 2.11
|
||||
private JavaSparkContext jsc;
|
||||
|
||||
private boolean enableSupportedVersionCheck;
|
||||
|
||||
public SparkInterpreter(Properties property) {
|
||||
super(property);
|
||||
|
|
@ -609,6 +608,9 @@ public class SparkInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void open() {
|
||||
this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean(
|
||||
property.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true"));
|
||||
|
||||
// set properties and do login before creating any spark stuff for secured cluster
|
||||
if (isYarnMode()) {
|
||||
System.setProperty("SPARK_YARN_MODE", "true");
|
||||
|
|
@ -1153,12 +1155,16 @@ public class SparkInterpreter extends Interpreter {
|
|||
return obj;
|
||||
}
|
||||
|
||||
boolean isUnsupportedSparkVersion() {
|
||||
return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion();
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret a single line.
|
||||
*/
|
||||
@Override
|
||||
public InterpreterResult interpret(String line, InterpreterContext context) {
|
||||
if (sparkVersion.isUnsupportedVersion()) {
|
||||
if (isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark " + sparkVersion.toString()
|
||||
+ " is not supported");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -108,6 +108,10 @@ public class SparkRInterpreter extends Interpreter {
|
|||
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
sparkInterpreter.populateSparkWebUrl(interpreterContext);
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
||||
String jobGroup = Utils.buildJobGroupId(interpreterContext);
|
||||
sparkInterpreter.getSparkContext().setJobGroup(jobGroup, "Zeppelin", false);
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ public class SparkSqlInterpreter extends Interpreter {
|
|||
SQLContext sqlc = null;
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion()) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@
|
|||
"propertyName": "spark.master",
|
||||
"defaultValue": "local[*]",
|
||||
"description": "Spark master uri. ex) spark://masterhost:7077"
|
||||
},
|
||||
"zeppelin.spark.unSupportedVersionCheck": {
|
||||
"envName": null,
|
||||
"propertyName": "zeppelin.spark.enableSupportedVersionCheck",
|
||||
"defaultValue": "true",
|
||||
"description": "Do not change - developer only setting, not for production use"
|
||||
}
|
||||
},
|
||||
"editor": {
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@
|
|||
"propertyName": "spark.master",
|
||||
"defaultValue": "local[*]",
|
||||
"description": "Spark master uri. ex) spark://masterhost:7077"
|
||||
},
|
||||
"zeppelin.spark.unSupportedVersionCheck": {
|
||||
"envName": null,
|
||||
"propertyName": "zeppelin.spark.enableSupportedVersionCheck",
|
||||
"defaultValue": "true",
|
||||
"description": "Do not change - developer only setting, not for production use"
|
||||
}
|
||||
},
|
||||
"editor": {
|
||||
|
|
|
|||
|
|
@ -19,11 +19,7 @@ package org.apache.zeppelin.display;
|
|||
|
||||
import java.io.Serializable;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.zeppelin.display.Input.ParamOption;
|
||||
|
||||
|
|
@ -33,7 +29,7 @@ import org.apache.zeppelin.display.Input.ParamOption;
|
|||
public class GUI implements Serializable {
|
||||
|
||||
Map<String, Object> params = new HashMap<>(); // form parameters from client
|
||||
Map<String, Input> forms = new TreeMap<>(); // form configuration
|
||||
LinkedHashMap<String, Input> forms = new LinkedHashMap<>(); // form configuration
|
||||
|
||||
public GUI() {
|
||||
|
||||
|
|
@ -47,11 +43,11 @@ public class GUI implements Serializable {
|
|||
return params;
|
||||
}
|
||||
|
||||
public Map<String, Input> getForms() {
|
||||
public LinkedHashMap<String, Input> getForms() {
|
||||
return forms;
|
||||
}
|
||||
|
||||
public void setForms(Map<String, Input> forms) {
|
||||
public void setForms(LinkedHashMap<String, Input> forms) {
|
||||
this.forms = forms;
|
||||
}
|
||||
|
||||
|
|
@ -105,6 +101,6 @@ public class GUI implements Serializable {
|
|||
}
|
||||
|
||||
public void clear() {
|
||||
this.forms = new TreeMap<>();
|
||||
this.forms = new LinkedHashMap<>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,13 +20,7 @@ package org.apache.zeppelin.display;
|
|||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -291,21 +285,21 @@ public class Input implements Serializable {
|
|||
return new Input(varName, displayName, type, arg, defaultValue, paramOptions, hidden);
|
||||
}
|
||||
|
||||
public static Map<String, Input> extractSimpleQueryParam(String script) {
|
||||
Map<String, Input> params = new HashMap<>();
|
||||
public static LinkedHashMap<String, Input> extractSimpleQueryForm(String script) {
|
||||
LinkedHashMap<String, Input> forms = new LinkedHashMap<>();
|
||||
if (script == null) {
|
||||
return params;
|
||||
return forms;
|
||||
}
|
||||
String replaced = script;
|
||||
|
||||
Matcher match = VAR_PTN.matcher(replaced);
|
||||
while (match.find()) {
|
||||
Input param = getInputForm(match);
|
||||
params.put(param.name, param);
|
||||
Input form = getInputForm(match);
|
||||
forms.put(form.name, form);
|
||||
}
|
||||
|
||||
params.remove("pql");
|
||||
return params;
|
||||
forms.remove("pql");
|
||||
return forms;
|
||||
}
|
||||
|
||||
private static final String DEFAULT_DELIMITER = ",";
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ public class InputTest {
|
|||
public void testFormExtraction() {
|
||||
// input form
|
||||
String script = "${input_form=}";
|
||||
Map<String, Input> forms = Input.extractSimpleQueryParam(script);
|
||||
Map<String, Input> forms = Input.extractSimpleQueryForm(script);
|
||||
assertEquals(1, forms.size());
|
||||
Input form = forms.get("input_form");
|
||||
assertEquals("input_form", form.name);
|
||||
|
|
@ -54,13 +54,13 @@ public class InputTest {
|
|||
|
||||
// input form with display name & default value
|
||||
script = "${input_form(Input Form)=xxx}";
|
||||
forms = Input.extractSimpleQueryParam(script);
|
||||
forms = Input.extractSimpleQueryForm(script);
|
||||
form = forms.get("input_form");
|
||||
assertEquals("xxx", form.defaultValue);
|
||||
|
||||
// selection form
|
||||
script = "${select_form(Selection Form)=op1,op1|op2(Option 2)|op3}";
|
||||
form = Input.extractSimpleQueryParam(script).get("select_form");
|
||||
form = Input.extractSimpleQueryForm(script).get("select_form");
|
||||
assertEquals("select_form", form.name);
|
||||
assertEquals("op1", form.defaultValue);
|
||||
assertArrayEquals(new ParamOption[]{new ParamOption("op1", null),
|
||||
|
|
@ -68,7 +68,7 @@ public class InputTest {
|
|||
|
||||
// checkbox form
|
||||
script = "${checkbox:checkbox_form=op1,op1|op2|op3}";
|
||||
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
|
||||
form = Input.extractSimpleQueryForm(script).get("checkbox_form");
|
||||
assertEquals("checkbox_form", form.name);
|
||||
assertEquals("checkbox", form.type);
|
||||
assertArrayEquals(new Object[]{"op1"}, (Object[]) form.defaultValue);
|
||||
|
|
@ -77,7 +77,7 @@ public class InputTest {
|
|||
|
||||
// checkbox form with multiple default checks
|
||||
script = "${checkbox:checkbox_form(Checkbox Form)=op1|op3,op1(Option 1)|op2|op3}";
|
||||
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
|
||||
form = Input.extractSimpleQueryForm(script).get("checkbox_form");
|
||||
assertEquals("checkbox_form", form.name);
|
||||
assertEquals("Checkbox Form", form.displayName);
|
||||
assertEquals("checkbox", form.type);
|
||||
|
|
@ -87,7 +87,7 @@ public class InputTest {
|
|||
|
||||
// checkbox form with no default check
|
||||
script = "${checkbox:checkbox_form(Checkbox Form)=,op1(Option 1)|op2(Option 2)|op3(Option 3)}";
|
||||
form = Input.extractSimpleQueryParam(script).get("checkbox_form");
|
||||
form = Input.extractSimpleQueryForm(script).get("checkbox_form");
|
||||
assertEquals("checkbox_form", form.name);
|
||||
assertEquals("Checkbox Form", form.displayName);
|
||||
assertEquals("checkbox", form.type);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -480,4 +481,29 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
|
|||
int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]);
|
||||
return version;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testZeppelinContextDynamicForms() throws IOException {
|
||||
Note note = ZeppelinServer.notebook.createNote(anonymous);
|
||||
Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
|
||||
note.setName("note");
|
||||
Map config = p.getConfig();
|
||||
config.put("enabled", true);
|
||||
p.setConfig(config);
|
||||
String code = "%spark.spark z.input(\"my_input\", \"default_name\")\n" +
|
||||
"z.select(\"my_select\", \"select_2\"," +
|
||||
"Seq((\"1\", \"select_1\"), (\"2\", \"select_2\")))\n" +
|
||||
"z.checkbox(\"my_checkbox\", Seq(\"check_1\"), " +
|
||||
"Seq((\"1\", \"check_1\"), (\"2\", \"check_2\")))";
|
||||
p.setText(code);
|
||||
p.setAuthenticationInfo(anonymous);
|
||||
note.run(p.getId());
|
||||
waitForFinish(p);
|
||||
|
||||
assertEquals(Status.FINISHED, p.getStatus());
|
||||
Iterator<String> formIter = p.settings.getForms().keySet().iterator();
|
||||
assert(formIter.next().equals("my_input"));
|
||||
assert(formIter.next().equals("my_select"));
|
||||
assert(formIter.next().equals("my_checkbox"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ limitations under the License.
|
|||
ng-show="!paragraph.config.tableHide"
|
||||
class=" paragraphForm form-horizontal row">
|
||||
<div class="form-group col-sm-6 col-md-6 col-lg-4"
|
||||
ng-repeat="formulaire in paragraph.settings.forms | toArray | orderBy:'name.toString()'"
|
||||
ng-repeat="formulaire in paragraph.settings.forms | toArray"
|
||||
ng-init="loadForm(formulaire, paragraph.settings.params)">
|
||||
<label class="control-label input-sm" ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }">{{formulaire.name}}</label>
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -593,7 +593,9 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
+ "org.apache.zeppelin.hbase.HbaseInterpreter,"
|
||||
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter,"
|
||||
+ "org.apache.zeppelin.beam.BeamInterpreter,"
|
||||
+ "org.apache.zeppelin.scio.ScioInterpreter"),
|
||||
+ "org.apache.zeppelin.scio.ScioInterpreter,"
|
||||
+ "org.apache.zeppelin.groovy.GroovyInterpreter"
|
||||
),
|
||||
ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
|
||||
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
|
||||
ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
|
||||
|
|
@ -603,7 +605,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
|
||||
ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
|
||||
+ "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
|
||||
+ "scalding,jdbc,hbase,bigquery,beam,pig,scio"),
|
||||
+ "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy"),
|
||||
ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
|
||||
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
|
||||
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
|
||||
|
|
|
|||
|
|
@ -21,11 +21,7 @@ import static java.lang.String.format;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
|
@ -320,8 +316,8 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
interpreterSettingManager);
|
||||
|
||||
Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
|
||||
Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());
|
||||
Map<String, Input> form = new HashMap<>(srcParagraph.settings.getForms());
|
||||
Map<String, Object> param = srcParagraph.settings.getParams();
|
||||
LinkedHashMap<String, Input> form = srcParagraph.settings.getForms();
|
||||
|
||||
newParagraph.setConfig(config);
|
||||
newParagraph.settings.setParams(param);
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
|
|||
public Paragraph cloneParagraphForUser(String user) {
|
||||
Paragraph p = new Paragraph();
|
||||
p.settings.setParams(Maps.newHashMap(settings.getParams()));
|
||||
p.settings.setForms(Maps.newHashMap(settings.getForms()));
|
||||
p.settings.setForms(Maps.newLinkedHashMap(settings.getForms()));
|
||||
p.setConfig(Maps.newHashMap(config));
|
||||
p.setTitle(getTitle());
|
||||
p.setText(getText());
|
||||
|
|
@ -389,8 +389,8 @@ public class Paragraph extends Job implements Serializable, Cloneable {
|
|||
settings.clear();
|
||||
} else if (repl.getFormType() == FormType.SIMPLE) {
|
||||
String scriptBody = getScriptBody();
|
||||
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
|
||||
// from script body
|
||||
// inputs will be built from script body
|
||||
LinkedHashMap<String, Input> inputs = Input.extractSimpleQueryForm(scriptBody);
|
||||
|
||||
final AngularObjectRegistry angularRegistry =
|
||||
repl.getInterpreterGroup().getAngularObjectRegistry();
|
||||
|
|
|
|||
Loading…
Reference in a new issue