diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 526b414782..25b263abea 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -14,7 +14,9 @@ First time? Check out the contributing guide - https://zeppelin.apache.org/contr * Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] ### How should this be tested? -Outline the steps to test the PR here. +* First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration +* Strongly recommended: add automated unit tests for any new or changed behavior +* Outline any manual steps to test the PR here. ### Screenshots (if appropriate) diff --git a/README.md b/README.md index e12d2aedb5..126520a3a5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Apache Zeppelin -**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)
-**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)
+**Documentation:** [User Guide](https://zeppelin.apache.org/docs/latest/index.html)
+**Mailing Lists:** [User and Dev mailing list](https://zeppelin.apache.org/community.html)
**Continuous Integration:** [![Build Status](https://travis-ci.org/apache/zeppelin.svg?branch=master)](https://travis-ci.org/apache/zeppelin)
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)
@@ -15,15 +15,15 @@ Core feature: * Built-in Apache Spark support -To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org) +To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](https://zeppelin.apache.org) ## Getting Started ### Install binary package -Please go to [install](http://zeppelin.apache.org/docs/snapshot/install/install.html) to install Apache Zeppelin from binary package. +Please go to [install](https://zeppelin.apache.org/docs/latest/install/install.html) to install Apache Zeppelin from binary package. ### Build from source -Please check [Build from source](http://zeppelin.apache.org/docs/snapshot/install/build.html) to build Zeppelin from source. +Please check [Build from source](https://zeppelin.apache.org/docs/latest/install/build.html) to build Zeppelin from source. diff --git a/alluxio/pom.xml b/alluxio/pom.xml index ed6b981286..f36494e94c 100644 --- a/alluxio/pom.xml +++ b/alluxio/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -34,6 +34,7 @@ 1.0.0 + alluxio @@ -129,54 +130,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/alluxio - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/alluxio - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/angular/pom.xml b/angular/pom.xml index be43e496a4..9ff2acb80e 100644 --- a/angular/pom.xml +++ b/angular/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -32,6 +32,10 @@ 0.8.0-SNAPSHOT Zeppelin: Angular interpreter + + angular + + ${project.groupId} @@ -61,54 +65,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/angular - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/angular - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/beam/pom.xml b/beam/pom.xml index 166652793f..a1d4761535 100644 --- a/beam/pom.xml +++ b/beam/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -41,6 +41,7 @@ 4.1.1.Final 3.1.0 1.3 + beam @@ -240,69 +241,18 @@ + - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/beam - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/beam - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - + + maven-resources-plugin + diff --git a/bigquery/pom.xml b/bigquery/pom.xml index f974b98815..db88edafaf 100644 --- a/bigquery/pom.xml +++ b/bigquery/pom.xml @@ -21,9 +21,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -41,6 +42,7 @@ v2-rev265-1.21.0 2.6 + bigquery @@ -99,12 +101,12 @@ maven-enforcer-plugin - - - enforce - none - - + + + maven-dependency-plugin + + + maven-resources-plugin @@ -116,63 +118,22 @@ - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/bqsql - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/bqsql - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + maven-assembly-plugin + + + + + org.apache.zeppelin.bigquery.BigQueryInterpreter + + + + + jar-with-dependencies + + - - maven-assembly-plugin - - - - - org.apache.zeppelin.bigquery.BigQueryInterpreter - - - - - jar-with-dependencies - - - + diff --git a/bin/common.sh b/bin/common.sh index d425cb1ab1..6447ec8daf 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -122,7 +122,7 @@ JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties" export JAVA_OPTS JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}" -if [[ -z "${SPARK_YARN_CLUSTER}" ]]; then +if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties" else JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties" diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 5245e25608..d27b0767bf 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -148,6 +148,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}" + export HADOOP_CONF_DIR=${HADOOP_CONF_DIR} else # autodetect HADOOP_CONF_HOME by heuristic if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then diff --git a/cassandra/pom.xml b/cassandra/pom.xml index 05108e9fd4..4b139e9c60 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -20,10 +20,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -49,6 +49,7 @@ 2.15.2 1.0 1.7.1 + cassandra @@ -241,55 +242,14 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/cassandra - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/cassandra - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin + diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index f1bfb61038..b25ba19c80 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -181,13 +181,13 @@ - zeppelin.hdfs.keytab + zeppelin.server.kerberos.keytab keytab for accessing kerberized hdfs - zeppelin.hdfs.principal + zeppelin.server.kerberos.principal principal for accessing kerberized hdfs diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index ecdccbd7ff..215c94426a 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -137,6 +137,7 @@
  • Lens
  • Livy
  • Markdown
    • +
  • Neo4j
  • Pig
  • Postgresql, HAWQ
  • R
    • diff --git a/docs/assets/themes/zeppelin/img/docs-img/neo4j-config.png b/docs/assets/themes/zeppelin/img/docs-img/neo4j-config.png new file mode 100644 index 0000000000..2de3699e8a Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/neo4j-config.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png b/docs/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png new file mode 100644 index 0000000000..177e0a5e76 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/neo4j-graph.png b/docs/assets/themes/zeppelin/img/docs-img/neo4j-graph.png new file mode 100644 index 0000000000..396b960db6 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/neo4j-graph.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif b/docs/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif new file mode 100644 index 0000000000..28c191516f Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif differ diff --git a/docs/development/writing_zeppelin_interpreter.md b/docs/development/writing_zeppelin_interpreter.md index 6ba24bc44a..f4c0bc939d 100644 --- a/docs/development/writing_zeppelin_interpreter.md +++ b/docs/development/writing_zeppelin_interpreter.md @@ -40,7 +40,49 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se ## Make your own Interpreter Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods. -You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter. +For your interpreter project, you need to make `interpreter-parent` as your parent project and use plugin `maven-enforcer-plugin`, `maven-dependency-plugin` and `maven-resources-plugin`. Here's one sample pom.xml + +``` + + 4.0.0 + + + interpreter-parent + org.apache.zeppelin + 0.8.0-SNAPSHOT + ../interpreter-parent + + + ... + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + + + + + + maven-enforcer-plugin + + + maven-dependency-plugin + + + maven-resources-plugin + + + + + +``` + +You should include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` as your interpreter's dependency in `pom.xml`. Bes +And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter. There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar. diff --git a/docs/index.md b/docs/index.md index 5e991f1903..dbec0402d9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -142,6 +142,7 @@ limitations under the License. * [Lens](./interpreter/lens.html) * [Livy](./interpreter/livy.html) * [markdown](./interpreter/markdown.html) + * [Neo4j](./interpreter/neo4j.html) * [Pig](./interpreter/pig.html) * [Postgresql, HAWQ](./interpreter/postgresql.html) * [Python](./interpreter/python.html) diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 1741a80c8b..09bf6e1c27 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -144,7 +144,12 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` zeppelin.livy.ssl.trustStorePassword password for trustStore file. Used when livy ssl is enabled - + + + zeppelin.livy.http.headers + key_1: value_1; key_2: value_2 + custom http headers when calling livy rest api. Each http header is separated by `;`, and each header is one key value pair where key value is separated by `:` + **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.** diff --git a/docs/interpreter/neo4j.md b/docs/interpreter/neo4j.md new file mode 100644 index 0000000000..37f1f8c935 --- /dev/null +++ b/docs/interpreter/neo4j.md @@ -0,0 +1,117 @@ +--- +layout: page +title: "Neo4j Interpreter for Apache Zeppelin" +description: "Neo4j is a native graph database, designed to store and process graphs from bottom to top." +group: interpreter +--- + +{% include JB/setup %} + +# Neo4j Interpreter for Apache Zeppelin + +
    + +## Overview +[Neo4j](https://neo4j.com/product/) is a native graph database, designed to store and process graphs from bottom to top. + + +![Neo4j - Interpreter - Video]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif) + +## Configuration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    PropertyDefaultDescription
    neo4j.urlbolt://localhost:7687The Neo4j's BOLT url.
    neo4j.auth.typeBASICThe Neo4j's authentication type (NONE, BASIC).
    neo4j.auth.userneo4jThe Neo4j user name.
    neo4j.auth.passwordneo4jThe Neo4j user password.
    neo4j.max.concurrency50Max concurrency call from Zeppelin to Neo4j server.
    + +
    + ![Interpreter configuration]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-config.png) +
    + + +## Enabling the Neo4j Interpreter +In a notebook, to enable the **Neo4j** interpreter, click the **Gear** icon and select **Neo4j**. + +## Using the Neo4j Interpreter +In a paragraph, use `%neo4j` to select the Neo4j interpreter and then input the Cypher commands. +For list of Cypher commands please refer to the official [Cyper Refcard](http://neo4j.com/docs/cypher-refcard/current/) + +```bash +%neo4j +//Sample the TrumpWorld dataset +WITH +'https://docs.google.com/spreadsheets/u/1/d/1Z5Vo5pbvxKJ5XpfALZXvCzW26Cl4we3OaN73K9Ae5Ss/export?format=csv&gid=1996904412' AS url +LOAD CSV WITH HEADERS FROM url AS row +RETURN row.`Entity A`, row.`Entity A Type`, row.`Entity B`, row.`Entity B Type`, row.Connection, row.`Source(s)` +LIMIT 10 +``` + +The Neo4j interpreter leverages the [Network display system](../usage/display_system/basic.html#network) allowing to visualize the them directly from the paragraph. + + +### Write your Cypher queries and navigate your graph + +This query: + +```bash +%neo4j +MATCH (vp:Person {name:"VLADIMIR PUTIN"}), (dt:Person {name:"DONALD J. TRUMP"}) +MATCH path = allShortestPaths( (vp)-[*]-(dt) ) +RETURN path +``` +produces the following result_ +![Neo4j - Graph - Result]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-graph.png) + +### Apply Zeppelin Dynamic Forms +You can leverage [Zeppelin Dynamic Form](../usage/dynamic_form/intro.html) inside your queries. This query: + +```bash +%neo4j +MATCH (o:Organization)-[r]-() +RETURN o.name, count(*), collect(distinct type(r)) AS types +ORDER BY count(*) DESC +LIMIT ${Show top=10} +``` + +produces the following result: +![Neo4j - Zeppelin - Dynamic Forms]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png) + diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index be5b3e5286..bbd9065faa 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -181,6 +181,7 @@ For example, * **local[*]** in local mode * **spark://master:7077** in standalone cluster * **yarn-client** in Yarn client mode + * **yarn-cluster** in Yarn cluster mode * **mesos://host:5050** in Mesos cluster That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. @@ -188,6 +189,11 @@ For the further information about Spark & Zeppelin version compatibility, please > Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile. +### 3. Yarn mode +Zeppelin support both yarn client and yarn cluster mode (yarn cluster mode is supported from 0.8.0). For yarn mode, you must specify `SPARK_HOME` & `HADOOP_CONF_DIR`. +You can either specify them in `zeppelin-env.sh`, or in interpreter setting page. Specifying them in `zeppelin-env.sh` means you can use only one version of `spark` & `hadoop`. Specifying them +in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance. + ## SparkContext, SQLContext, SparkSession, ZeppelinContext SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments. Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x. diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index e91a7df2ba..21ae5b3a60 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -77,7 +77,19 @@ If both are defined, then the **environment variables** will take priority. * Enables a way to specify a ',' separated list of allowed origins for REST and websockets.
    e.g. http://localhost:8080 - + +
    ZEPPELIN_CREDENTIALS_PERSIST
    +
    zeppelin.credentials.persist
    + true + Persist credentials on a JSON file (credentials.json) + + +
    ZEPPELIN_CREDENTIALS_ENCRYPT_KEY
    +
    zeppelin.credentials.encryptKey
    + + If provided, encrypt passwords on the credentials.json file (passwords will be stored as plain-text otherwise + + N/A
    zeppelin.anonymous.allowed
    true @@ -411,6 +423,20 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order ``` +### Storing user credentials + +In order to avoid having to re-enter credentials every time you restart/redeploy Zeppelin, you can store the user credentials. Zeppelin supports this via the ZEPPELIN_CREDENTIALS_PERSIST configuration. + +Please notice that passwords will be stored in *plain text* by default. To encrypt the passwords, use the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY config variable. This will encrypt passwords using the AES-128 algorithm. + +You can generate an appropriate encryption key any way you'd like - for instance, by using the openssl tool: + +``` +openssl enc -aes-128-cbc -k secret -P -md sha1 +``` + +*Important*: storing your encryption key in a configuration file is _not advised_. Depending on your environment security needs, you may want to consider utilizing a credentials server, storing the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY as an OS env variable, or any other approach that would not colocate the encryption key and the encrypted content (the credentials.json file). + ### Obfuscating Passwords using the Jetty Password Tool diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md index 0b65f91661..f6b8b5c08d 100644 --- a/docs/setup/storage/storage.md +++ b/docs/setup/storage/storage.md @@ -57,7 +57,7 @@ To enable versioning for all your local notebooks though a standard Git reposito ## Notebook Storage in hadoop compatible file system repository Notes may be stored in hadoop compatible file system such as hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `FileSystemNotebookRepo`, then `zeppelin.notebook.dir` is the path on the hadoop compatible file system. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files. -If your hadoop cluster is kerberized, then you need to specify `zeppelin.hdfs.keytab` and `zeppelin.hdfs.principal` +If your hadoop cluster is kerberized, then you need to specify `zeppelin.server.kerberos.keytab` and `zeppelin.server.kerberos.principal` ``` diff --git a/docs/usage/rest_api/notebook.md b/docs/usage/rest_api/notebook.md index ff935531ae..9879ad999f 100644 --- a/docs/usage/rest_api/notebook.md +++ b/docs/usage/rest_api/notebook.md @@ -752,6 +752,59 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete, +
    + +### Update paragraph + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    DescriptionThis ```PUT``` method update paragraph contents using given id, e.g. {"text": "hello"} +
    URL```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/paragraph/[paragraphId]```
    Success code200
    Bad Request code400
    Forbidden code403
    Not Found code404
    Fail code500
    sample JSON input
    +{
+  "title": "Hello world",
+  "text": "println(\"hello world\")"
+}
    sample JSON response
    +{
+  "status": "OK",
+  "message": ""
+  }
+}
    +
    ### Update paragraph configuration diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index 6042a14eae..b317e28581 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent zeppelin-elasticsearch @@ -32,6 +32,7 @@ Zeppelin: Elasticsearch interpreter + elasticsearch 2.4.3 4.0.2 18.0 @@ -93,54 +94,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/elasticsearch - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/elasticsearch - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java index 33448df3db..6251b92512 100644 --- a/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java +++ b/elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java @@ -112,7 +112,7 @@ public class ElasticsearchInterpreter extends Interpreter { @Override public void open() { - logger.info("Properties: {}", getProperty()); + logger.info("Properties: {}", getProperties()); String clientType = getProperty(ELASTICSEARCH_CLIENT_TYPE); clientType = clientType == null ? null : clientType.toLowerCase(); @@ -123,15 +123,15 @@ public class ElasticsearchInterpreter extends Interpreter { catch (final NumberFormatException e) { this.resultSize = 10; logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " + - property.get(ELASTICSEARCH_RESULT_SIZE), e); + getProperty(ELASTICSEARCH_RESULT_SIZE), e); } try { if (StringUtils.isEmpty(clientType) || "transport".equals(clientType)) { - elsClient = new TransportBasedClient(getProperty()); + elsClient = new TransportBasedClient(getProperties()); } else if ("http".equals(clientType)) { - elsClient = new HttpBasedClient(getProperty()); + elsClient = new HttpBasedClient(getProperties()); } else { logger.error("Unknown type of Elasticsearch client: " + clientType); diff --git a/file/pom.xml b/file/pom.xml index 2493c1fae4..1a76683215 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -20,9 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -33,11 +34,13 @@ + file 2.0 2.22.2 2.18.1 + file @@ -79,62 +82,14 @@ - - org.apache.maven.plugins - maven-surefire-plugin - ${plugin.surefire.version} - - maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/file - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/file - false - false - true - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java index d7aad192ee..cf83672734 100644 --- a/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java @@ -20,6 +20,7 @@ package org.apache.zeppelin.file; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; @@ -86,7 +87,7 @@ public abstract class FileInterpreter extends Interpreter { // Functions that each file system implementation must override - public abstract String listAll(String path); + public abstract String listAll(String path) throws InterpreterException; public abstract boolean isDirectory(String path); diff --git a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java index 244101c9bd..d715ed93a8 100644 --- a/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java +++ b/file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java @@ -202,7 +202,7 @@ public class HDFSFileInterpreter extends FileInterpreter { return "No such File or directory"; } - public String listAll(String path) { + public String listAll(String path) throws InterpreterException { String all = ""; if (exceptionOnConnect != null) return "Error connecting to provided endpoint."; diff --git a/flink/pom.xml b/flink/pom.xml index 19e7c5e48b..5c0cc3bf90 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -35,6 +35,7 @@ + flink 1.1.3 2.3.7 2.0.1 @@ -280,68 +281,16 @@ - - maven-enforcer-plugin - - - enforce - none - - - + + maven-enforcer-plugin + + + maven-dependency-plugin + + + maven-resources-plugin + - - org.apache.maven.plugins - maven-surefire-plugin - - 1 - false - -Xmx1024m -XX:MaxPermSize=256m - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/flink - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/flink - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - diff --git a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java index 710eace666..19c77de914 100644 --- a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java +++ b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java @@ -17,7 +17,6 @@ */ package org.apache.zeppelin.flink; -import java.lang.reflect.InvocationTargetException; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; @@ -34,10 +33,8 @@ import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.flink.runtime.util.EnvironmentInformation; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; @@ -46,11 +43,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Console; -import scala.None; -import scala.Option; import scala.Some; import scala.collection.JavaConversions; -import scala.collection.immutable.Nil; import scala.concurrent.duration.FiniteDuration; import scala.runtime.AbstractFunction0; import scala.tools.nsc.Settings; @@ -80,7 +74,7 @@ public class FlinkInterpreter extends Interpreter { public void open() { out = new ByteArrayOutputStream(); flinkConf = new org.apache.flink.configuration.Configuration(); - Properties intpProperty = getProperty(); + Properties intpProperty = getProperties(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); diff --git a/geode/pom.xml b/geode/pom.xml index e8eb9fc846..4a9dadd234 100644 --- a/geode/pom.xml +++ b/geode/pom.xml @@ -21,9 +21,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -34,6 +35,7 @@ + geode 1.1.0 1.3 @@ -86,54 +88,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/geode - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/geode - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/groovy/pom.xml b/groovy/pom.xml index bee50bd82e..64c51f8561 100644 --- a/groovy/pom.xml +++ b/groovy/pom.xml @@ -20,11 +20,11 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -33,6 +33,10 @@ 0.8.0-SNAPSHOT Zeppelin: Groovy interpreter + + groovy + + ${project.groupId} @@ -67,81 +71,14 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - true - - - -Xlint:unchecked - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - maven-enforcer-plugin - 1.3.1 - - - enforce - none - - - maven-dependency-plugin - 2.8 - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/groovy - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/groovy - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java b/groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java index 7f6809a899..babda8f760 100644 --- a/groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java +++ b/groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java @@ -172,7 +172,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport { /** * starts or continues rendering html/angular and returns MarkupBuilder to build html. * 
     g.html().with{
-   * 	h1("hello")
+   *  h1("hello")
    *  h2("world")
    * }
    */ @@ -316,12 +316,12 @@ public class GObject extends groovy.lang.GroovyObjectSupport { @ZeppelinApi public void run(String noteId, String paragraphId, InterpreterContext context) { if (paragraphId.equals(context.getParagraphId())) { - throw new InterpreterException("Can not run current Paragraph"); + throw new RuntimeException("Can not run current Paragraph"); } List runners = getInterpreterContextRunner(noteId, paragraphId, context); if (runners.size() <= 0) { - throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size()); + throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size()); } for (InterpreterContextRunner r : runners) { r.run(); @@ -338,7 +338,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport { List runners = getInterpreterContextRunner(noteId, context); if (runners.size() <= 0) { - throw new InterpreterException("Note " + noteId + " not found " + runners.size()); + throw new RuntimeException("Note " + noteId + " not found " + runners.size()); } for (InterpreterContextRunner r : runners) { diff --git a/groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java b/groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java index e10828368f..01e97e6bdd 100644 --- a/groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java +++ b/groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java @@ -17,8 +17,6 @@ package org.apache.zeppelin.groovy; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.StringWriter; import java.io.PrintWriter; import java.io.File; @@ -26,10 +24,8 @@ import java.util.*; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Scheduler; @@ -40,7 +36,6 @@ import org.slf4j.LoggerFactory; import groovy.lang.GroovyShell; import groovy.lang.Script; import org.codehaus.groovy.control.CompilerConfiguration; -import org.codehaus.groovy.runtime.ResourceGroovyMethods; import org.codehaus.groovy.runtime.StackTraceUtils; import java.util.concurrent.ConcurrentHashMap; @@ -167,7 +162,7 @@ public class GroovyInterpreter extends Interpreter { //put shared bindings evaluated in this interpreter bindings.putAll(sharedBindings); //put predefined bindings - bindings.put("g", new GObject(log, out, this.getProperty(), contextInterpreter, bindings)); + bindings.put("g", new GObject(log, out, this.getProperties(), contextInterpreter, bindings)); bindings.put("out", new PrintWriter(out, true)); script.run(); @@ -204,7 +199,7 @@ public class GroovyInterpreter extends Interpreter { Thread t = (Thread) object; t.dumpStack(); t.interrupt(); - //t.stop(); //TODO: need some way to terminate maybe through GObject.. + //t.stop(); //TODO(dlukyanov): need some way to terminate maybe through GObject.. } catch (Throwable t) { log.error("Failed to cancel script: " + t, t); } diff --git a/hbase/pom.xml b/hbase/pom.xml index 08b0cd7071..8f27631ea2 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -20,9 +20,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -33,6 +34,7 @@ + hbase 1.0.0 2.6.0 1.6.8 @@ -115,55 +117,14 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/hbase - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/hbase - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin + diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 74d3ed1aec..63c1928334 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -68,7 +68,7 @@ public class HbaseInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON); this.writer = new StringWriter(); scriptingContainer.setOutput(this.writer); @@ -88,7 +88,7 @@ public class HbaseInterpreter extends Interpreter { } logger.info("Absolute Ruby Source:" + abs_ruby_src.toString()); - // hirb.rb:41 requires the following system property to be set. + // hirb.rb:41 requires the following system properties to be set. Properties sysProps = System.getProperties(); sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString()); diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java index 38a8b4d1e0..53040f91a5 100644 --- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -15,6 +15,7 @@ package org.apache.zeppelin.hbase; import org.apache.log4j.BasicConfigurator; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.BeforeClass; import org.junit.Test; @@ -35,7 +36,7 @@ public class HbaseInterpreterTest { private static HbaseInterpreter hbaseInterpreter; @BeforeClass - public static void setUp() throws NullPointerException { + public static void setUp() throws NullPointerException, InterpreterException { BasicConfigurator.configure(); Properties properties = new Properties(); properties.put("hbase.home", ""); diff --git a/helium-dev/pom.xml b/helium-dev/pom.xml index 77c4dee8e0..8514946d76 100644 --- a/helium-dev/pom.xml +++ b/helium-dev/pom.xml @@ -23,8 +23,9 @@ org.apache.zeppelin - zeppelin + interpreter-parent 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -32,6 +33,10 @@ 0.8.0-SNAPSHOT Zeppelin: Helium development interpreter + + helium-dev + + org.apache.zeppelin @@ -43,46 +48,14 @@ + + maven-enforcer-plugin + maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/helium-dev - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/helium-dev - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java index 7d1c361de7..ba1a564dd7 100644 --- a/helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java +++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java @@ -74,7 +74,8 @@ public class DevInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { this.context = context; try { return interpreterEvent.interpret(st, context); diff --git a/ignite/pom.xml b/ignite/pom.xml index 9b1855c6ff..ccbd37d8e4 100644 --- a/ignite/pom.xml +++ b/ignite/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent zeppelin-ignite_2.10 @@ -32,6 +32,7 @@ Zeppelin: Apache Ignite interpreter + ignite 2.2.0 @@ -106,55 +107,12 @@ maven-enforcer-plugin - - - enforce - none - - - - org.apache.maven.plugins maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/ignite - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/ignite - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java index 41803bb31b..6af8eb549e 100644 --- a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java +++ b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java @@ -93,7 +93,7 @@ public class IgniteSqlInterpreter extends Interpreter { } @Override - public void close() { + public void close() throws InterpreterException { try { if (conn != null) { conn.close(); diff --git a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java index b06d457c38..a6ae0ea6a5 100644 --- a/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java +++ b/ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java @@ -27,6 +27,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; @@ -82,7 +83,7 @@ public class IgniteSqlInterpreterTest { } @After - public void tearDown() { + public void tearDown() throws InterpreterException { intp.close(); ignite.close(); } diff --git a/interpreter-parent/pom.xml b/interpreter-parent/pom.xml new file mode 100644 index 0000000000..32bdc570e7 --- /dev/null +++ b/interpreter-parent/pom.xml @@ -0,0 +1,138 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + .. + + + org.apache.zeppelin + interpreter-parent + pom + 0.8.0-SNAPSHOT + Zeppelin: Interpreter Parent + + + + + ${project.groupId} + zeppelin-interpreter + 0.8.0-SNAPSHOT + provided + + + + junit + junit + ${junit.version} + test + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + + + + + + + maven-enforcer-plugin + + + enforce + none + + + + + + maven-dependency-plugin + + + copy-interpreter-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/${interpreter.name} + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/${interpreter.name} + false + false + true + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + maven-resources-plugin + + + copy-interpreter-setting + package + + resources + + + ${project.build.directory}/../../interpreter/${interpreter.name} + + + + + + + + + diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 71d33109e6..beec50c0ad 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -21,9 +21,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -34,6 +35,7 @@ + jdbc 9.4-1201-jdbc41 2.7.2 1.4.190 @@ -158,54 +160,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/jdbc - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/jdbc - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 2c5258cfd7..f3dfddcbfd 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -45,8 +45,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.apache.thrift.transport.TTransportException; -import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -172,7 +170,7 @@ public class JDBCInterpreter extends KerberosInterpreter { @Override public void open() { super.open(); - for (String propertyKey : property.stringPropertyNames()) { + for (String propertyKey : properties.stringPropertyNames()) { logger.debug("propertyKey: {}", propertyKey); String[] keyValue = propertyKey.split("\\.", 2); if (2 == keyValue.length) { @@ -185,7 +183,7 @@ public class JDBCInterpreter extends KerberosInterpreter { prefixProperties = new Properties(); basePropretiesMap.put(keyValue[0].trim(), prefixProperties); } - prefixProperties.put(keyValue[1].trim(), property.getProperty(propertyKey)); + prefixProperties.put(keyValue[1].trim(), getProperty(propertyKey)); } } @@ -211,8 +209,8 @@ public class JDBCInterpreter extends KerberosInterpreter { protected boolean isKerboseEnabled() { - if (!isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) { - UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property); + if (!isEmpty(getProperty("zeppelin.jdbc.auth.type"))) { + UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(properties); if (authType.equals(KERBEROS)) { return true; } @@ -356,7 +354,7 @@ public class JDBCInterpreter extends KerberosInterpreter { } private void setUserProperty(String propertyKey, InterpreterContext interpreterContext) - throws SQLException, IOException { + throws SQLException, IOException, InterpreterException { String user = interpreterContext.getAuthenticationInfo().getUser(); @@ -424,18 +422,19 @@ public class JDBCInterpreter extends KerberosInterpreter { final Properties properties = jdbcUserConfigurations.getPropertyMap(propertyKey); final String url = properties.getProperty(URL_KEY); - if (isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) { + if (isEmpty(getProperty("zeppelin.jdbc.auth.type"))) { connection = getConnectionFromPool(url, user, propertyKey, properties); } else { - UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property); + UserGroupInformation.AuthenticationMethod authType = + JDBCSecurityImpl.getAuthtype(getProperties()); final String connectionUrl = appendProxyUserToURL(url, user, propertyKey); - JDBCSecurityImpl.createSecureConfiguration(property, authType); + JDBCSecurityImpl.createSecureConfiguration(getProperties(), authType); switch (authType) { case KERBEROS: if (user == null || "false".equalsIgnoreCase( - property.getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) { + getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) { connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties); } else { if (basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) { @@ -497,7 +496,7 @@ public class JDBCInterpreter extends KerberosInterpreter { return connectionUrl.toString(); } - private String getPassword(Properties properties) throws IOException { + private String getPassword(Properties properties) throws IOException, InterpreterException { if (isNotEmpty(properties.getProperty(PASSWORD_KEY))) { return properties.getProperty(PASSWORD_KEY); } else if (isNotEmpty(properties.getProperty(JDBC_JCEKS_FILE)) @@ -850,7 +849,7 @@ public class JDBCInterpreter extends KerberosInterpreter { @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) throws InterpreterException { List candidates = new ArrayList<>(); String propertyKey = getPropertyKey(buf); String sqlCompleterKey = diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index e6f9598f8b..1b3f045ce6 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -37,6 +37,7 @@ import java.util.Properties; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.FIFOScheduler; @@ -349,7 +350,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { } @Test - public void testAutoCompletion() throws SQLException, IOException { + public void testAutoCompletion() throws SQLException, IOException, InterpreterException { Properties properties = new Properties(); properties.setProperty("common.max_count", "1000"); properties.setProperty("common.max_retry", "3"); diff --git a/kylin/pom.xml b/kylin/pom.xml index c3559cd04b..b70facbcc1 100644 --- a/kylin/pom.xml +++ b/kylin/pom.xml @@ -21,9 +21,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent 4.0.0 @@ -33,6 +34,10 @@ 0.8.0-SNAPSHOT Zeppelin: Kylin interpreter + + kylin + + @@ -61,57 +66,14 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/kylin - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/kylin - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin - diff --git a/lens/pom.xml b/lens/pom.xml index 0328355793..ecc0a12ed6 100644 --- a/lens/pom.xml +++ b/lens/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -34,6 +34,7 @@ + lens 2.5.0-beta 1.1.0.RELEASE 2.4.0 @@ -163,69 +164,13 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/lens - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/lens - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - - org.apache.maven.plugins - maven-clean-plugin - - - - ${basedir}/../interpreter/lens - false - - - - - + + maven-resources-plugin + diff --git a/livy/pom.xml b/livy/pom.xml index e7217fb9da..0ec174a9c4 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -22,10 +22,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -36,6 +36,7 @@ + livy 1.3 4.3.0.RELEASE 1.0.1.RELEASE @@ -193,56 +194,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/livy - - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/livy - - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index ccab09bfce..03a269eeae 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -62,6 +62,8 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** @@ -80,6 +82,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { protected boolean displayAppInfo; protected LivyVersion livyVersion; private RestTemplate restTemplate; + private Map customHeaders = new HashMap<>(); Set paragraphsToCancel = Collections.newSetFromMap( new ConcurrentHashMap()); @@ -96,12 +99,39 @@ public abstract class BaseLivyInterpreter extends Interpreter { this.pullStatusInterval = Integer.parseInt( property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + "")); this.restTemplate = createRestTemplate(); + if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) { + String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";"); + for (String header : headers) { + String[] splits = header.split(":", -1); + if (splits.length != 2) { + throw new RuntimeException("Invalid format of http headers: " + header + + ", valid http header format is HEADER_NAME:HEADER_VALUE"); + } + customHeaders.put(splits[0].trim(), envSubstitute(splits[1].trim())); + } + } + } + + private String envSubstitute(String value) { + String newValue = new String(value); + Pattern pattern = Pattern.compile("\\$\\{(.*)\\}"); + Matcher matcher = pattern.matcher(value); + while (matcher.find()) { + String env = matcher.group(1); + newValue = newValue.replace("${" + env + "}", System.getenv(env)); + } + return newValue; + } + + // only for testing + Map getCustomHeaders() { + return customHeaders; } public abstract String getSessionKind(); @Override - public void open() { + public void open() throws InterpreterException { try { initLivySession(); } catch (LivyException e) { @@ -198,7 +228,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { throws LivyException { try { Map conf = new HashMap<>(); - for (Map.Entry entry : property.entrySet()) { + for (Map.Entry entry : getProperties().entrySet()) { if (entry.getKey().toString().startsWith("livy.spark.") && !entry.getValue().toString().isEmpty()) conf.put(entry.getKey().toString().substring(5), entry.getValue().toString()); @@ -428,15 +458,15 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate createRestTemplate() { - String keytabLocation = property.getProperty("zeppelin.livy.keytab"); - String principal = property.getProperty("zeppelin.livy.principal"); + String keytabLocation = getProperty("zeppelin.livy.keytab"); + String principal = getProperty("zeppelin.livy.principal"); boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal); HttpClient httpClient = null; if (livyURL.startsWith("https:")) { - String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); - String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword"); + String keystoreFile = getProperty("zeppelin.livy.ssl.trustStore"); + String password = getProperty("zeppelin.livy.ssl.trustStorePassword"); if (StringUtils.isBlank(keystoreFile)) { throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl"); } @@ -523,6 +553,9 @@ public abstract class BaseLivyInterpreter extends Interpreter { HttpHeaders headers = new HttpHeaders(); headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE); headers.add("X-Requested-By", "zeppelin"); + for (Map.Entry entry : customHeaders.entrySet()) { + headers.add(entry.getKey(), entry.getValue()); + } ResponseEntity response = null; try { if (method.equals("POST")) { diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 20d044811b..486e31c2c7 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -59,7 +59,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter { } @Override - public void open() { + public void open() throws InterpreterException { this.sparkInterpreter = getSparkInterpreter(); // As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession // to judge whether it is using spark2. @@ -93,7 +93,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter { } } - private LivySparkInterpreter getSparkInterpreter() { + private LivySparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; LivySparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName()); diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 9a0aef481e..d4131345f1 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -75,8 +75,9 @@ public class LivyInterpreterIT { return true; } + // @Test - public void testSparkInterpreterRDD() { + public void testSparkInterpreterRDD() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -195,8 +196,9 @@ public class LivyInterpreterIT { } } + // @Test - public void testSparkInterpreterDataFrame() { + public void testSparkInterpreterDataFrame() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -284,7 +286,7 @@ public class LivyInterpreterIT { } // @Test - public void testSparkSQLInterpreter() { + public void testSparkSQLInterpreter() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -319,7 +321,7 @@ public class LivyInterpreterIT { // @Test - public void testSparkSQLCancellation() { + public void testSparkSQLCancellation() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -400,7 +402,7 @@ public class LivyInterpreterIT { } // @Test - public void testStringWithTruncation() { + public void testStringWithTruncation() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -459,8 +461,9 @@ public class LivyInterpreterIT { } } + // @Test - public void testStringWithoutTruncation() { + public void testStringWithoutTruncation() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -525,7 +528,7 @@ public class LivyInterpreterIT { } @Test - public void testPySparkInterpreter() throws LivyException { + public void testPySparkInterpreter() throws LivyException, InterpreterException { if (!checkPreCondition()) { return; } @@ -645,7 +648,7 @@ public class LivyInterpreterIT { } // @Test - public void testSparkInterpreterWithDisplayAppInfo() { + public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException { if (!checkPreCondition()) { return; } @@ -684,7 +687,7 @@ public class LivyInterpreterIT { } // @Test - public void testSparkRInterpreter() throws LivyException { + public void testSparkRInterpreter() throws LivyException, InterpreterException { if (!checkPreCondition()) { return; } @@ -756,7 +759,7 @@ public class LivyInterpreterIT { } // @Test - public void testLivyTutorialNote() throws IOException { + public void testLivyTutorialNote() throws IOException, InterpreterException { if (!checkPreCondition()) { return; } diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java index fdef9b13d1..24d70ec2b0 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java @@ -39,9 +39,17 @@ public class LivySQLInterpreterTest { properties.setProperty("zeppelin.livy.url", "http://localhost:8998"); properties.setProperty("zeppelin.livy.session.create_timeout", "120"); properties.setProperty("zeppelin.livy.spark.sql.maxResult", "3"); + properties.setProperty("zeppelin.livy.http.headers", "HEADER_1: VALUE_1_${HOME}"); sqlInterpreter = new LivySparkSQLInterpreter(properties); } + @Test + public void testHttpHeaders() { + assertEquals(1, sqlInterpreter.getCustomHeaders().size()); + assertTrue(sqlInterpreter.getCustomHeaders().get("HEADER_1").startsWith("VALUE_1_")); + assertNotEquals("VALUE_1_${HOME}", sqlInterpreter.getCustomHeaders().get("HEADER_1")); + } + @Test public void testParseSQLOutput() { // Empty sql output diff --git a/markdown/pom.xml b/markdown/pom.xml index 9675aa7095..ca95449807 100644 --- a/markdown/pom.xml +++ b/markdown/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -33,6 +33,7 @@ Zeppelin: Markdown interpreter + md 3.4 2.2-cj-1.0 1.6.0 @@ -85,54 +86,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/md - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/md - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/neo4j/pom.xml b/neo4j/pom.xml new file mode 100644 index 0000000000..298726fb82 --- /dev/null +++ b/neo4j/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-neo4j + jar + 0.8.0-SNAPSHOT + Zeppelin: Neo4j interpreter + + + 1.4.3 + 3.2.3 + 3.2.3 + 2.8.9 + + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + org.neo4j.driver + neo4j-java-driver + ${neo4j.driver.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + org.neo4j.test + neo4j-harness + ${neo4j.version} + test + + + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/neo4j + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/neo4j + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + + diff --git a/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jConnectionManager.java b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jConnectionManager.java new file mode 100644 index 0000000000..7cd504ef20 --- /dev/null +++ b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jConnectionManager.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.graph.neo4j; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourcePool; +import org.neo4j.driver.v1.AuthToken; +import org.neo4j.driver.v1.AuthTokens; +import org.neo4j.driver.v1.Config; +import org.neo4j.driver.v1.Driver; +import org.neo4j.driver.v1.GraphDatabase; +import org.neo4j.driver.v1.Session; +import org.neo4j.driver.v1.StatementResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Neo4j connection manager for Zeppelin. + */ +public class Neo4jConnectionManager { + static final Logger LOGGER = LoggerFactory.getLogger(Neo4jConnectionManager.class); + + public static final String NEO4J_SERVER_URL = "neo4j.url"; + public static final String NEO4J_AUTH_TYPE = "neo4j.auth.type"; + public static final String NEO4J_AUTH_USER = "neo4j.auth.user"; + public static final String NEO4J_AUTH_PASSWORD = "neo4j.auth.password"; + public static final String NEO4J_MAX_CONCURRENCY = "neo4j.max.concurrency"; + + private static final Pattern PROPERTY_PATTERN = Pattern.compile("\\{\\w+\\}"); + private static final String REPLACE_CURLY_BRACKETS = "\\{|\\}"; + + private static final Pattern $_PATTERN = Pattern.compile("\\$\\w+\\}"); + private static final String REPLACE_$ = "\\$"; + + private Driver driver = null; + + private final String neo4jUrl; + + private final Config config; + + private final AuthToken authToken; + + /** + * + * Enum type for the AuthToken + * + */ + public enum Neo4jAuthType {NONE, BASIC} + + public Neo4jConnectionManager(Properties properties) { + this.neo4jUrl = properties.getProperty(NEO4J_SERVER_URL); + this.config = Config.build() + .withMaxIdleSessions(Integer.parseInt(properties.getProperty(NEO4J_MAX_CONCURRENCY))) + .toConfig(); + String authType = properties.getProperty(NEO4J_AUTH_TYPE); + switch (Neo4jAuthType.valueOf(authType.toUpperCase())) { + case BASIC: + String username = properties.getProperty(NEO4J_AUTH_USER); + String password = properties.getProperty(NEO4J_AUTH_PASSWORD); + this.authToken = AuthTokens.basic(username, password); + break; + case NONE: + LOGGER.debug("Creating NONE authentication"); + this.authToken = AuthTokens.none(); + break; + default: + throw new RuntimeException("Neo4j authentication type not supported"); + } + } + + private Driver getDriver() { + if (driver == null) { + driver = GraphDatabase.driver(this.neo4jUrl, this.authToken, this.config); + } + return driver; + } + + public void open() { + getDriver(); + } + + public void close() { + getDriver().close(); + } + + private Session getSession() { + return getDriver().session(); + } + + public StatementResult execute(String cypherQuery, + InterpreterContext interpreterContext) { + Map params = new HashMap<>(); + if (interpreterContext != null) { + ResourcePool resourcePool = interpreterContext.getResourcePool(); + Set keys = extractParams(cypherQuery, PROPERTY_PATTERN, REPLACE_CURLY_BRACKETS); + keys.addAll(extractParams(cypherQuery, $_PATTERN, REPLACE_$)); + for (String key : keys) { + Resource resource = resourcePool.get(key); + if (resource != null) { + params.put(key, resource.get()); + } + } + } + LOGGER.debug("Executing cypher query {} with params {}", cypherQuery, params); + StatementResult result; + try (Session session = getSession()) { + result = params.isEmpty() + ? getSession().run(cypherQuery) : getSession().run(cypherQuery, params); + } + return result; + } + + public StatementResult execute(String cypherQuery) { + return execute(cypherQuery, null); + } + + private Set extractParams(String cypherQuery, Pattern pattern, String replaceChar) { + Matcher matcher = pattern.matcher(cypherQuery); + Set keys = new HashSet<>(); + while (matcher.find()) { + keys.add(matcher.group().replaceAll(replaceChar, StringUtils.EMPTY)); + } + return keys; + } + +} diff --git a/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreter.java b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreter.java new file mode 100644 index 0000000000..a625522558 --- /dev/null +++ b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreter.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.graph.neo4j; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.graph.neo4j.utils.Neo4jConversionUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.graph.GraphResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.neo4j.driver.internal.types.InternalTypeSystem; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.driver.v1.Record; +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.driver.v1.Value; +import org.neo4j.driver.v1.types.Node; +import org.neo4j.driver.v1.types.Relationship; +import org.neo4j.driver.v1.types.TypeSystem; +import org.neo4j.driver.v1.util.Pair; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Neo4j interpreter for Zeppelin. + */ +public class Neo4jCypherInterpreter extends Interpreter { + private static final String TABLE = "%table"; + public static final String NEW_LINE = "\n"; + public static final String TAB = "\t"; + + private static final String MAP_KEY_TEMPLATE = "%s.%s"; + + private Map labels; + + private Set types; + + private final Neo4jConnectionManager neo4jConnectionManager; + + private final ObjectMapper jsonMapper = new ObjectMapper(); + + public Neo4jCypherInterpreter(Properties properties) { + super(properties); + this.neo4jConnectionManager = new Neo4jConnectionManager(properties); + } + + @Override + public void open() { + this.neo4jConnectionManager.open(); + } + + @Override + public void close() { + this.neo4jConnectionManager.close(); + } + + public Map getLabels(boolean refresh) { + if (labels == null || refresh) { + Map old = labels == null ? + new LinkedHashMap() : new LinkedHashMap<>(labels); + labels = new LinkedHashMap<>(); + StatementResult result = this.neo4jConnectionManager.execute("CALL db.labels()"); + Set colors = new HashSet<>(); + while (result.hasNext()) { + Record record = result.next(); + String label = record.get("label").asString(); + String color = old.get(label); + while (color == null || colors.contains(color)) { + color = Neo4jConversionUtils.getRandomLabelColor(); + } + colors.add(color); + labels.put(label, color); + } + } + return labels; + } + + private Set getTypes(boolean refresh) { + if (types == null || refresh) { + types = new HashSet<>(); + StatementResult result = this.neo4jConnectionManager.execute("CALL db.relationshipTypes()"); + while (result.hasNext()) { + Record record = result.next(); + types.add(record.get("relationshipType").asString()); + } + } + return types; + } + + @Override + public InterpreterResult interpret(String cypherQuery, InterpreterContext interpreterContext) { + logger.info("Opening session"); + if (StringUtils.isBlank(cypherQuery)) { + return new InterpreterResult(Code.SUCCESS); + } + try { + StatementResult result = this.neo4jConnectionManager.execute(cypherQuery, + interpreterContext); + Set nodes = new HashSet<>(); + Set relationships = new HashSet<>(); + List columns = new ArrayList<>(); + List> lines = new ArrayList>(); + while (result.hasNext()) { + Record record = result.next(); + List> fields = record.fields(); + List line = new ArrayList<>(); + for (Pair field : fields) { + if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.NODE())) { + nodes.add(field.value().asNode()); + } else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.RELATIONSHIP())) { + relationships.add(field.value().asRelationship()); + } else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.PATH())) { + nodes.addAll(Iterables.asList(field.value().asPath().nodes())); + relationships.addAll(Iterables.asList(field.value().asPath().relationships())); + } else { + setTabularResult(field.key(), field.value(), columns, line, + InternalTypeSystem.TYPE_SYSTEM); + } + } + if (!line.isEmpty()) { + lines.add(line); + } + } + if (!nodes.isEmpty()) { + return renderGraph(nodes, relationships); + } else { + return renderTable(columns, lines); + } + } catch (Exception e) { + logger.error("Exception while interpreting cypher query", e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + } + + private void setTabularResult(String key, Object obj, List columns, List line, + TypeSystem typeSystem) { + if (obj instanceof Value) { + Value value = (Value) obj; + if (value.hasType(typeSystem.MAP())) { + Map map = value.asMap(); + for (Entry entry : map.entrySet()) { + setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(), + columns, line, typeSystem); + } + } else { + addValueToLine(key, columns, line, value); + } + } else if (obj instanceof Map) { + Map map = (Map) obj; + for (Entry entry : map.entrySet()) { + setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(), + columns, line, typeSystem); + } + } else { + addValueToLine(key, columns, line, obj); + } + } + + private void addValueToLine(String key, List columns, List line, Object value) { + if (!columns.contains(key)) { + columns.add(key); + } + int position = columns.indexOf(key); + if (line.size() < columns.size()) { + for (int i = line.size(); i < columns.size(); i++) { + line.add(null); + } + } + if (value != null) { + if (value instanceof Value) { + Value val = (Value) value; + if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.LIST())) { + value = val.asList(); + } else if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.MAP())) { + value = val.asMap(); + } + } + if (value instanceof Collection) { + try { + value = jsonMapper.writer().writeValueAsString(value); + } catch (Exception ignored) {} + } + } + line.set(position, value == null ? null : value.toString()); + } + + private InterpreterResult renderTable(List cols, List> lines) { + logger.info("Executing renderTable method"); + StringBuilder msg = null; + if (cols.isEmpty()) { + msg = new StringBuilder(); + } else { + msg = new StringBuilder(TABLE); + msg.append(NEW_LINE); + msg.append(StringUtils.join(cols, TAB)); + msg.append(NEW_LINE); + for (List line : lines) { + if (line.size() < cols.size()) { + for (int i = line.size(); i < cols.size(); i++) { + line.add(null); + } + } + msg.append(StringUtils.join(line, TAB)); + msg.append(NEW_LINE); + } + } + return new InterpreterResult(Code.SUCCESS, msg.toString()); + } + + private InterpreterResult renderGraph(Set nodes, + Set relationships) { + logger.info("Executing renderGraph method"); + List nodesList = new ArrayList<>(); + List relsList = new ArrayList<>(); + for (Relationship rel : relationships) { + relsList.add(Neo4jConversionUtils.toZeppelinRelationship(rel)); + } + Map labels = getLabels(true); + for (Node node : nodes) { + nodesList.add(Neo4jConversionUtils.toZeppelinNode(node, labels)); + } + return new GraphResult(Code.SUCCESS, + new GraphResult.Graph(nodesList, relsList, labels, getTypes(true), true)); + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton() + .createOrGetParallelScheduler(Neo4jCypherInterpreter.class.getName() + this.hashCode(), + Integer.parseInt(getProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY))); + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public void cancel(InterpreterContext context) { + } + +} diff --git a/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/utils/Neo4jConversionUtils.java b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/utils/Neo4jConversionUtils.java new file mode 100644 index 0000000000..484940198a --- /dev/null +++ b/neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/utils/Neo4jConversionUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.graph.neo4j.utils; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import org.neo4j.driver.v1.types.Node; +import org.neo4j.driver.v1.types.Relationship; + +/** + * Neo4jConversionUtils + */ +public class Neo4jConversionUtils { + private Neo4jConversionUtils() {} + + private static final String[] LETTERS = "0123456789ABCDEF".split(""); + + public static final String COLOR_GREY = "#D3D3D3"; + + public static org.apache.zeppelin.tabledata.Node toZeppelinNode(Node n, + Map graphLabels) { + Set labels = new LinkedHashSet<>(); + String firstLabel = null; + for (String label : n.labels()) { + if (firstLabel == null) { + firstLabel = label; + } + labels.add(label); + } + return new org.apache.zeppelin.tabledata.Node(n.id(), n.asMap(), + labels); + } + + public static org.apache.zeppelin.tabledata.Relationship + toZeppelinRelationship(Relationship r) { + return new org.apache.zeppelin.tabledata.Relationship(r.id(), r.asMap(), + r.startNodeId(), r.endNodeId(), r.type()); + } + + public static String getRandomLabelColor() { + char[] color = new char[7]; + color[0] = '#'; + for (int i = 1; i < color.length; i++) { + color[i] = LETTERS[(int) Math.floor(Math.random() * 16)].charAt(0); + } + return new String(color); + } + +} diff --git a/neo4j/src/main/resources/interpreter-setting.json b/neo4j/src/main/resources/interpreter-setting.json new file mode 100644 index 0000000000..8db4367cc3 --- /dev/null +++ b/neo4j/src/main/resources/interpreter-setting.json @@ -0,0 +1,42 @@ +[ + { + "group": "neo4j", + "name": "neo4j", + "className": "org.apache.zeppelin.graph.neo4j.Neo4jCypherInterpreter", + "properties": { + "neo4j.url": { + "envName": null, + "propertyName": "neo4j.url", + "defaultValue": "bolt://localhost:7687", + "description": "The Neo4j's BOLT url." + }, + "neo4j.auth.type": { + "envName": null, + "propertyName": "neo4j.auth.type", + "defaultValue": "BASIC", + "description": "The Neo4j's authentication type (NONE, BASIC)." + }, + "neo4j.auth.user": { + "envName": null, + "propertyName": "neo4j.auth.user", + "defaultValue": "", + "description": "The Neo4j user name." + }, + "neo4j.auth.password": { + "envName": null, + "propertyName": "neo4j.auth.password", + "defaultValue": "", + "description": "The Neo4j user password." + }, + "neo4j.max.concurrency": { + "envName": null, + "propertyName": "neo4j.max.concurrency", + "defaultValue": "50", + "description": "Max concurrency call from Zeppelin to Neo4j server." + } + }, + "editor": { + "editOnDblClick": false + } + } +] diff --git a/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java new file mode 100644 index 0000000000..1bb14b71e1 --- /dev/null +++ b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.graph.neo4j; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.graph.GraphResult; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.neo4j.harness.ServerControls; +import org.neo4j.harness.TestServerBuilders; + +import com.google.gson.Gson; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class Neo4jCypherInterpreterTest { + + private Neo4jCypherInterpreter interpreter; + + private InterpreterContext context; + + private static ServerControls server; + + private static final Gson gson = new Gson(); + + private static final String LABEL_PERSON = "Person"; + private static final String REL_KNOWS = "KNOWS"; + + private static final String CYPHER_FOREACH = "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))"; + private static final String CHPHER_UNWIND = "UNWIND range(1,1000) as x " + + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) " + + "CREATE (n)-[:%s]->(m)"; + + @BeforeClass + public static void setUpNeo4jServer() throws Exception { + server = TestServerBuilders.newInProcessBuilder() + .withConfig("dbms.security.auth_enabled","false") + .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10")) + .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS)) + .newServer(); + } + + @AfterClass + public static void tearDownNeo4jServer() throws Exception { + server.close(); + } + + @Before + public void setUpZeppelin() { + Properties p = new Properties(); + p.setProperty(Neo4jConnectionManager.NEO4J_SERVER_URL, server.boltURI().toString()); + p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString()); + p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50"); + interpreter = new Neo4jCypherInterpreter(p); + context = new InterpreterContext("note", "id", null, "title", "text", + new AuthenticationInfo(), + new HashMap(), + new GUI(), + new AngularObjectRegistry(new InterpreterGroup().getId(), null), + new LocalResourcePool("id"), + new LinkedList(), + new InterpreterOutput(null)); + } + + @After + public void tearDownZeppelin() throws Exception { + interpreter.close(); + } + + @Test + public void testTableWithArray() { + interpreter.open(); + InterpreterResult result = interpreter.interpret("return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context); + assertEquals(Code.SUCCESS, result.code()); + final String tableResult = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[1,2,3]\n"; + assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY)); + + result = interpreter.interpret("return 'a' as colA, 'b' as colB, [{key: \"value\"}, {key: 1}] as colC", context); + assertEquals(Code.SUCCESS, result.code()); + final String tableResultWithMap = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n"; + assertEquals(tableResultWithMap, result.toString().replace("%table ", StringUtils.EMPTY)); + } + + @Test + public void testCreateIndex() { + interpreter.open(); + InterpreterResult result = interpreter.interpret("CREATE INDEX ON :Person(name)", context); + assertEquals(Code.SUCCESS, result.code()); + assertEquals(StringUtils.EMPTY, result.toString()); + } + + @Test + public void testRenderTable() { + interpreter.open(); + InterpreterResult result = interpreter.interpret("MATCH (n:Person) " + + "WHERE n.name IN ['name1', 'name2', 'name3'] " + + "RETURN n.name AS name, n.age AS age", context); + assertEquals(Code.SUCCESS, result.code()); + final String tableResult = "name\tage\n\"name1\"\t1\n\"name2\"\t2\n\"name3\"\t3\n"; + assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY)); + } + + @Test + public void testRenderMap() { + interpreter.open(); + final String jsonQuery = "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object"; + final String objectKey = "object.key"; + final String objectListKey = "object.listKey"; + InterpreterResult result = interpreter.interpret(jsonQuery, context); + assertEquals(Code.SUCCESS, result.code()); + String[] rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE); + assertEquals(rows.length, 2); + List header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB)); + assertEquals(header.contains(objectKey), true); + assertEquals(header.contains(objectListKey), true); + List row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value"); + assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + + final String query = "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}," + + "{key: \"value2\", listKey: [{inner: \"Map12\"}, {inner: \"Map22\"}]}] " + + "AS array UNWIND array AS object RETURN object"; + result = interpreter.interpret(query, context); + assertEquals(Code.SUCCESS, result.code()); + rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE); + assertEquals(rows.length, 3); + header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB)); + assertEquals(header.contains(objectKey), true); + assertEquals(header.contains(objectListKey), true); + row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value"); + assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value2"); + assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]"); + + final String jsonListWithNullQuery = "WITH [{key: \"value\", listKey: null}," + + "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] " + + "AS array UNWIND array AS object RETURN object"; + result = interpreter.interpret(jsonListWithNullQuery, context); + assertEquals(Code.SUCCESS, result.code()); + rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE); + assertEquals(rows.length, 3); + header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(header.contains(objectKey), true); + assertEquals(header.contains(objectListKey), true); + row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value"); + assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY); + assertEquals(row.get(header.indexOf(objectListKey)), ""); + row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value2"); + assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + + final String jsonListWithoutListKeyQuery = "WITH [{key: \"value\"}," + + "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] " + + "AS array UNWIND array AS object RETURN object"; + result = interpreter.interpret(jsonListWithoutListKeyQuery, context); + assertEquals(Code.SUCCESS, result.code()); + rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE); + assertEquals(rows.length, 3); + header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(header.contains(objectKey), true); + assertEquals(header.contains(objectListKey), true); + row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value"); + assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY); + row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1)); + assertEquals(row.size(), header.size()); + assertEquals(row.get(header.indexOf(objectKey)), "value2"); + assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + } + + @Test + public void testRenderNetwork() { + interpreter.open(); + InterpreterResult result = interpreter.interpret("MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context); + GraphResult.Graph graph = gson.fromJson(result.toString().replace("%network ", StringUtils.EMPTY), GraphResult.Graph.class); + assertEquals(2, graph.getNodes().size()); + assertEquals(true, graph.getNodes().iterator().next().getLabel().equals(LABEL_PERSON)); + assertEquals(1, graph.getEdges().size()); + assertEquals(true, graph.getEdges().iterator().next().getLabel().equals(REL_KNOWS)); + assertEquals(1, graph.getLabels().size()); + assertEquals(1, graph.getTypes().size()); + assertEquals(true, graph.getLabels().containsKey(LABEL_PERSON)); + assertEquals(REL_KNOWS, graph.getTypes().iterator().next()); + assertEquals(Code.SUCCESS, result.code()); + } + + @Test + public void testFallingQuery() { + interpreter.open(); + final String ERROR_MSG_EMPTY = ""; + InterpreterResult result = interpreter.interpret(StringUtils.EMPTY, context); + assertEquals(Code.SUCCESS, result.code()); + assertEquals(ERROR_MSG_EMPTY, result.toString()); + + result = interpreter.interpret(null, context); + assertEquals(Code.SUCCESS, result.code()); + assertEquals(ERROR_MSG_EMPTY, result.toString()); + + result = interpreter.interpret("MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", context); + assertEquals(Code.ERROR, result.code()); + } + +} diff --git a/pig/pom.xml b/pig/pom.xml index f76a3f90cc..4a56a34290 100644 --- a/pig/pom.xml +++ b/pig/pom.xml @@ -22,9 +22,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT + ../interpreter-parent org.apache.zeppelin @@ -36,6 +37,7 @@ http://zeppelin.apache.org + pig 0.17.0 2.6.0 0.7.0 @@ -175,58 +177,13 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/pig - - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/pig - - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - + + maven-resources-plugin + maven-surefire-plugin diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java index 893741651f..0f2d59bb9e 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.pig; import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.PigServer; import org.apache.pig.impl.logicalLayer.FrontendException; @@ -60,7 +59,7 @@ public class PigInterpreter extends BasePigInterpreter { } try { pigServer = new PigServer(execType); - for (Map.Entry entry : getProperty().entrySet()) { + for (Map.Entry entry : getProperties().entrySet()) { if (!entry.getKey().toString().startsWith("zeppelin.")) { pigServer.getPigContext().getProperties().setProperty(entry.getKey().toString(), entry.getValue().toString()); diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java index d3bc4325c4..da3d50e09b 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java @@ -55,7 +55,7 @@ public class PigQueryInterpreter extends BasePigInterpreter { } @Override - public void open() { + public void open() throws InterpreterException { pigServer = getPigInterpreter().getPigServer(); maxResult = Integer.parseInt(getProperty(MAX_RESULTS)); } @@ -159,7 +159,7 @@ public class PigQueryInterpreter extends BasePigInterpreter { return this.pigServer; } - private PigInterpreter getPigInterpreter() { + private PigInterpreter getPigInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; PigInterpreter pig = null; Interpreter p = getInterpreterInTheSameSessionByClassName(PigInterpreter.class.getName()); diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java index de297c75e9..f14cad6f27 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java @@ -21,6 +21,7 @@ package org.apache.zeppelin.pig; import org.apache.commons.io.IOUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.After; @@ -48,7 +49,7 @@ public class PigQueryInterpreterTest { private InterpreterContext context; @Before - public void setUp() { + public void setUp() throws InterpreterException { Properties properties = new Properties(); properties.put("zeppelin.pig.execType", "local"); properties.put("zeppelin.pig.maxResult", "20"); diff --git a/pom.xml b/pom.xml index acfcd0572c..7ff9acd7ae 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ bigquery alluxio scio + neo4j zeppelin-web zeppelin-server zeppelin-jupyter diff --git a/python/pom.xml b/python/pom.xml index d46cd10c2d..a906b5d5f5 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -33,6 +33,7 @@ Zeppelin: Python interpreter + python 0.9.2 **/PythonInterpreterWithPythonInstalledTest.java, @@ -141,17 +142,6 @@ - - maven-enforcer-plugin - 1.3.1 - - - enforce - none - - - - org.codehaus.mojo wagon-maven-plugin @@ -212,47 +202,14 @@ + + maven-enforcer-plugin + maven-dependency-plugin - 2.8 - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/python - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/python - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java index 40b9afdac4..05fe4ba47a 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java @@ -40,7 +40,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Random; +import java.security.SecureRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -55,7 +55,7 @@ public class IPythonClient { private final IPythonGrpc.IPythonBlockingStub blockingStub; private final IPythonGrpc.IPythonStub asyncStub; - private Random random = new Random(); + private SecureRandom random = new SecureRandom(); /** * Construct client for accessing RouteGuide server at {@code host:port}. diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index 193c3434c8..1c2ced501b 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -30,6 +30,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -112,19 +113,20 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } @Override - public void open() { + public void open() throws InterpreterException { try { if (ipythonClient != null) { // IPythonInterpreter might already been opened by PythonInterpreter return; } - pythonExecutable = getProperty().getProperty("zeppelin.python", "python"); + pythonExecutable = getProperty("zeppelin.python", "python"); LOGGER.info("Python Exec: " + pythonExecutable); + ipythonLaunchTimeout = Long.parseLong( - getProperty().getProperty("zeppelin.ipython.launch.timeout", "30000")); + getProperty("zeppelin.ipython.launch.timeout", "30000")); this.zeppelinContext = new PythonZeppelinContext( getInterpreterGroup().getInterpreterHookRegistry(), - Integer.parseInt(getProperty().getProperty("zeppelin.python.maxResult", "1000"))); + Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000"))); int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); LOGGER.info("Launching IPython Kernel at port: " + ipythonPort); @@ -243,16 +245,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } } - Map envs = EnvironmentUtils.getProcEnvironment(); - if (envs.containsKey("PYTHONPATH")) { - if (additionalPythonPath != null) { - envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH")); - } - } else { - envs.put("PYTHONPATH", additionalPythonPath); - } - - LOGGER.info("PYTHONPATH: " + envs.get("PYTHONPATH")); + Map envs = setupIPythonEnv(); executor.execute(cmd, envs, this); // wait until IPython kernel is started or timeout @@ -284,6 +277,18 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } } + protected Map setupIPythonEnv() throws IOException { + Map envs = EnvironmentUtils.getProcEnvironment(); + if (envs.containsKey("PYTHONPATH")) { + if (additionalPythonPath != null) { + envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH")); + } + } else { + envs.put("PYTHONPATH", additionalPythonPath); + } + return envs; + } + @Override public void close() { if (watchDog != null) { diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java index 0d122f5719..887beb8ce7 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java @@ -67,7 +67,8 @@ public class PythonCondaInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { InterpreterOutput out = context.out; Matcher activateMatcher = PATTERN_COMMAND_ACTIVATE.matcher(st); Matcher createMatcher = PATTERN_COMMAND_CREATE.matcher(st); @@ -126,7 +127,7 @@ public class PythonCondaInterpreter extends Interpreter { } private void changePythonEnvironment(String envName) - throws IOException, InterruptedException { + throws IOException, InterruptedException, InterpreterException { PythonInterpreter python = getPythonInterpreter(); String binPath = null; if (envName == null) { @@ -147,13 +148,13 @@ public class PythonCondaInterpreter extends Interpreter { python.setPythonCommand(binPath); } - private void restartPythonProcess() { + private void restartPythonProcess() throws InterpreterException { PythonInterpreter python = getPythonInterpreter(); python.close(); python.open(); } - protected PythonInterpreter getPythonInterpreter() { + protected PythonInterpreter getPythonInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; PythonInterpreter python = null; Interpreter p = @@ -213,7 +214,7 @@ public class PythonCondaInterpreter extends Interpreter { } private InterpreterResult runCondaActivate(String envName) - throws IOException, InterruptedException { + throws IOException, InterruptedException, InterpreterException { if (null == envName || envName.isEmpty()) { return new InterpreterResult(Code.ERROR, "Env name should be specified"); @@ -226,7 +227,7 @@ public class PythonCondaInterpreter extends Interpreter { } private InterpreterResult runCondaDeactivate() - throws IOException, InterruptedException { + throws IOException, InterruptedException, InterpreterException { changePythonEnvironment(null); restartPythonProcess(); @@ -375,10 +376,16 @@ public class PythonCondaInterpreter extends Interpreter { */ @Override public Scheduler getScheduler() { - PythonInterpreter pythonInterpreter = getPythonInterpreter(); - if (pythonInterpreter != null) { - return pythonInterpreter.getScheduler(); - } else { + PythonInterpreter pythonInterpreter = null; + try { + pythonInterpreter = getPythonInterpreter(); + if (pythonInterpreter != null) { + return pythonInterpreter.getScheduler(); + } else { + return null; + } + } catch (InterpreterException e) { + e.printStackTrace(); return null; } } diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java index cb0f62078f..22f6c2ee99 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java @@ -56,7 +56,8 @@ public class PythonDockerInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { File pythonScript = new File(getPythonInterpreter().getScriptPath()); InterpreterOutput out = context.out; @@ -105,7 +106,7 @@ public class PythonDockerInterpreter extends Interpreter { } - public void setPythonCommand(String cmd) { + public void setPythonCommand(String cmd) throws InterpreterException { PythonInterpreter python = getPythonInterpreter(); python.setPythonCommand(cmd); } @@ -140,21 +141,27 @@ public class PythonDockerInterpreter extends Interpreter { */ @Override public Scheduler getScheduler() { - PythonInterpreter pythonInterpreter = getPythonInterpreter(); - if (pythonInterpreter != null) { - return pythonInterpreter.getScheduler(); - } else { + PythonInterpreter pythonInterpreter = null; + try { + pythonInterpreter = getPythonInterpreter(); + if (pythonInterpreter != null) { + return pythonInterpreter.getScheduler(); + } else { + return null; + } + } catch (InterpreterException e) { + e.printStackTrace(); return null; } } - private void restartPythonProcess() { + private void restartPythonProcess() throws InterpreterException { PythonInterpreter python = getPythonInterpreter(); python.close(); python.open(); } - protected PythonInterpreter getPythonInterpreter() { + protected PythonInterpreter getPythonInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; PythonInterpreter python = null; Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName()); @@ -173,7 +180,7 @@ public class PythonDockerInterpreter extends Interpreter { return python; } - public boolean pull(InterpreterOutput out, String image) { + public boolean pull(InterpreterOutput out, String image) throws InterpreterException { int exit = 0; try { exit = runCommand(out, "docker", "pull", image); diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 50f6a8b9c6..4f897c8d6d 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -57,7 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import py4j.GatewayServer; -import py4j.commands.Command; /** * Python interpreter for Zeppelin. @@ -101,7 +100,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl File scriptFile = File.createTempFile("zeppelin_python-", ".py", new File("/tmp")); scriptPath = scriptFile.getAbsolutePath(); } catch (IOException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } } @@ -116,7 +115,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return path; } - private void createPythonScript() { + private void createPythonScript() throws InterpreterException { File out = new File(scriptPath); if (out.exists() && out.isDirectory()) { @@ -131,7 +130,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return scriptPath; } - private void copyFile(File out, String sourceFile) { + private void copyFile(File out, String sourceFile) throws InterpreterException { ClassLoader classLoader = getClass().getClassLoader(); try { FileOutputStream outStream = new FileOutputStream(out); @@ -144,7 +143,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } } - private void createGatewayServerAndStartScript() throws UnknownHostException { + private void createGatewayServerAndStartScript() + throws UnknownHostException, InterpreterException { createPythonScript(); if (System.getenv("ZEPPELIN_HOME") != null) { py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH; @@ -219,11 +219,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } @Override - public void open() { + public void open() throws InterpreterException { // try IPythonInterpreter first. If it is not available, we will fallback to the original // python interpreter implementation. iPythonInterpreter = getIPythonInterpreter(); - if (getProperty().getProperty("zeppelin.python.useIPython", "true").equals("true") && + if (getProperty("zeppelin.python.useIPython", "true").equals("true") && iPythonInterpreter.checkIPythonPrerequisite()) { try { iPythonInterpreter.open(); @@ -369,7 +369,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } @Override - public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { + public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) + throws InterpreterException { if (iPythonInterpreter != null) { return iPythonInterpreter.interpret(cmd, contextInterpreter); } @@ -551,7 +552,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl bootstrapCode += line + "\n"; } - interpret(bootstrapCode, context); + try { + interpret(bootstrapCode, context); + } catch (InterpreterException e) { + throw new IOException(e); + } } public GUI getGui() { diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java index e73d7b36bd..54984c3ce7 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java @@ -22,6 +22,7 @@ import java.util.Properties; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; @@ -42,7 +43,7 @@ public class PythonInterpreterPandasSql extends Interpreter { super(property); } - PythonInterpreter getPythonInterpreter() { + PythonInterpreter getPythonInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; PythonInterpreter python = null; Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName()); @@ -62,7 +63,7 @@ public class PythonInterpreterPandasSql extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { LOG.info("Open Python SQL interpreter instance: {}", this.toString()); try { @@ -76,14 +77,15 @@ public class PythonInterpreterPandasSql extends Interpreter { } @Override - public void close() { + public void close() throws InterpreterException { LOG.info("Close Python SQL interpreter instance: {}", this.toString()); Interpreter python = getPythonInterpreter(); python.close(); } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { LOG.info("Running SQL query: '{}' over Pandas DataFrame", st); Interpreter python = getPythonInterpreter(); diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 137d622bf8..6a388c2b9b 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.display.ui.Select; import org.apache.zeppelin.display.ui.TextBox; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -56,7 +57,7 @@ public class IPythonInterpreterTest { private IPythonInterpreter interpreter; @Before - public void setUp() { + public void setUp() throws InterpreterException { Properties properties = new Properties(); interpreter = new IPythonInterpreter(properties); InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class); @@ -71,11 +72,11 @@ public class IPythonInterpreterTest { @Test - public void testIPython() throws IOException, InterruptedException { + public void testIPython() throws IOException, InterruptedException, InterpreterException { testInterpreter(interpreter); } - public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException { + public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException, InterpreterException { // to make this test can run under both python2 and python3 InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -261,7 +262,11 @@ public class IPythonInterpreterTest { } catch (InterruptedException e) { e.printStackTrace(); } - interpreter.cancel(context2); + try { + interpreter.cancel(context2); + } catch (InterpreterException e) { + e.printStackTrace(); + } } }.start(); result = interpreter.interpret("import time\ntime.sleep(10)", context2); diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java index 897639627c..e6f5fca684 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java @@ -37,7 +37,7 @@ public class PythonCondaInterpreterTest { private PythonInterpreter python; @Before - public void setUp() { + public void setUp() throws InterpreterException { conda = spy(new PythonCondaInterpreter(new Properties())); python = mock(PythonInterpreter.class); @@ -57,7 +57,7 @@ public class PythonCondaInterpreterTest { } @Test - public void testListEnv() throws IOException, InterruptedException { + public void testListEnv() throws IOException, InterruptedException, InterpreterException { setMockCondaEnvList(); // list available env @@ -72,7 +72,7 @@ public class PythonCondaInterpreterTest { } @Test - public void testActivateEnv() throws IOException, InterruptedException { + public void testActivateEnv() throws IOException, InterruptedException, InterpreterException { setMockCondaEnvList(); String envname = "env1"; InterpreterContext context = getInterpreterContext(); @@ -84,7 +84,7 @@ public class PythonCondaInterpreterTest { } @Test - public void testDeactivate() { + public void testDeactivate() throws InterpreterException { InterpreterContext context = getInterpreterContext(); conda.interpret("deactivate", context); verify(python, times(1)).open(); diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java index 566b5e0b35..e5903949ee 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java @@ -41,7 +41,7 @@ public class PythonDockerInterpreterTest { private PythonInterpreter python; @Before - public void setUp() { + public void setUp() throws InterpreterException { docker = spy(new PythonDockerInterpreter(new Properties())); python = mock(PythonInterpreter.class); @@ -58,7 +58,7 @@ public class PythonDockerInterpreterTest { } @Test - public void testActivateEnv() { + public void testActivateEnv() throws InterpreterException { InterpreterContext context = getInterpreterContext(); docker.interpret("activate env", context); verify(python, times(1)).open(); @@ -68,7 +68,7 @@ public class PythonDockerInterpreterTest { } @Test - public void testDeactivate() { + public void testDeactivate() throws InterpreterException { InterpreterContext context = getInterpreterContext(); docker.interpret("deactivate", context); verify(python, times(1)).open(); diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java index d649e8979b..877e42881b 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java @@ -22,6 +22,7 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -83,7 +84,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene } @Test - public void dependenciesAreInstalled() { + public void dependenciesAreInstalled() throws InterpreterException { // matplotlib InterpreterResult ret = python.interpret("import matplotlib", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); @@ -94,7 +95,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene } @Test - public void showPlot() throws IOException { + public void showPlot() throws IOException, InterpreterException { // Simple plot test InterpreterResult ret; ret = python.interpret("import matplotlib.pyplot as plt", context); @@ -111,7 +112,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene @Test // Test for when configuration is set to auto-close figures after show(). - public void testClose() throws IOException { + public void testClose() throws IOException, InterpreterException { InterpreterResult ret; InterpreterResult ret1; InterpreterResult ret2; @@ -145,7 +146,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene @Test // Test for when configuration is set to not auto-close figures after show(). - public void testNoClose() throws IOException { + public void testNoClose() throws IOException, InterpreterException { InterpreterResult ret; InterpreterResult ret1; InterpreterResult ret2; diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java index 9e918c082f..5d667bd6bb 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java @@ -33,6 +33,7 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -107,18 +108,18 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener } @After - public void afterTest() throws IOException { + public void afterTest() throws IOException, InterpreterException { sql.close(); } @Test - public void dependenciesAreInstalled() { + public void dependenciesAreInstalled() throws InterpreterException { InterpreterResult ret = python.interpret("import pandas\nimport pandasql\nimport numpy\n", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); } @Test - public void errorMessageIfDependenciesNotInstalled() { + public void errorMessageIfDependenciesNotInstalled() throws InterpreterException { InterpreterResult ret; ret = sql.interpret("SELECT * from something", context); @@ -128,7 +129,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener } @Test - public void sqlOverTestDataPrintsTable() throws IOException { + public void sqlOverTestDataPrintsTable() throws IOException, InterpreterException { InterpreterResult ret; // given //String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34"; @@ -152,7 +153,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener } @Test - public void badSqlSyntaxFails() throws IOException { + public void badSqlSyntaxFails() throws IOException, InterpreterException { //when InterpreterResult ret = sql.interpret("select wrong syntax", context); @@ -162,7 +163,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener } @Test - public void showDataFrame() throws IOException { + public void showDataFrame() throws IOException, InterpreterException { InterpreterResult ret; ret = python.interpret("import pandas as pd", context); ret = python.interpret("import numpy as np", context); diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 195935d213..ef328f0bef 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -38,6 +38,7 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -64,7 +65,7 @@ public class PythonInterpreterTest implements InterpreterOutputListener { } @Before - public void beforeTest() throws IOException { + public void beforeTest() throws IOException, InterpreterException { cmdHistory = ""; // python interpreter @@ -96,20 +97,20 @@ public class PythonInterpreterTest implements InterpreterOutputListener { } @Test - public void testInterpret() throws InterruptedException, IOException { + public void testInterpret() throws InterruptedException, IOException, InterpreterException { InterpreterResult result = pythonInterpreter.interpret("print (\"hi\")", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); } @Test - public void testInterpretInvalidSyntax() throws IOException { + public void testInterpretInvalidSyntax() throws IOException, InterpreterException { InterpreterResult result = pythonInterpreter.interpret("for x in range(0,3): print (\"hi\")\n", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi")); } @Test - public void testRedefinitionZeppelinContext() { + public void testRedefinitionZeppelinContext() throws InterpreterException { String pyRedefinitionCode = "z = 1\n"; String pyRestoreCode = "z = __zeppelin__\n"; String pyValidCode = "z.input(\"test\")\n"; diff --git a/scalding/pom.xml b/scalding/pom.xml index cc1015f21b..763afe0bb4 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -33,6 +33,7 @@ Zeppelin: Scalding interpreter + scalding 2.6.0 0.16.1-RC1 @@ -147,55 +148,14 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/scalding - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/scalding - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + maven-resources-plugin + + org.scala-tools diff --git a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java index 7156c37e97..d3ebadaf05 100644 --- a/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java +++ b/scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java @@ -69,7 +69,7 @@ public class ScaldingInterpreter extends Interpreter { @Override public void open() { numOpenInstances = numOpenInstances + 1; - String maxOpenInstancesStr = property.getProperty(MAX_OPEN_INSTANCES, + String maxOpenInstancesStr = getProperty(MAX_OPEN_INSTANCES, MAX_OPEN_INSTANCES_DEFAULT); int maxOpenInstances = 50; try { @@ -83,8 +83,8 @@ public class ScaldingInterpreter extends Interpreter { return; } logger.info("Opening instance {}", numOpenInstances); - logger.info("property: {}", property); - String argsString = property.getProperty(ARGS_STRING, ARGS_STRING_DEFAULT); + logger.info("property: {}", getProperties()); + String argsString = getProperty(ARGS_STRING, ARGS_STRING_DEFAULT); String[] args; if (argsString == null) { args = new String[0]; @@ -121,7 +121,7 @@ public class ScaldingInterpreter extends Interpreter { return new InterpreterResult(Code.SUCCESS); } InterpreterResult interpreterResult = new InterpreterResult(Code.ERROR); - if (property.getProperty(ARGS_STRING).contains("hdfs")) { + if (getProperty(ARGS_STRING).contains("hdfs")) { UserGroupInformation ugi = null; try { ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); diff --git a/scio/pom.xml b/scio/pom.xml index 3d17dc88ef..27ccb0cc7b 100644 --- a/scio/pom.xml +++ b/scio/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -34,6 +34,7 @@ Zeppelin Scio support + scio 0.2.4 14.0.1 @@ -106,12 +107,12 @@ maven-enforcer-plugin - - - enforce - none - - + + + maven-dependency-plugin + + + maven-resources-plugin @@ -156,34 +157,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - - package - - copy - - - ${project.build.directory}/../../interpreter/scio - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - org.scala-tools maven-scala-plugin diff --git a/shell/pom.xml b/shell/pom.xml index 58d890084f..0157d86f53 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - zeppelin + interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT - .. + ../interpreter-parent org.apache.zeppelin @@ -33,17 +33,18 @@ Zeppelin: Shell interpreter + sh + 3.4 1.3 + sh ${project.groupId} zeppelin-interpreter - ${project.version} - provided @@ -79,54 +80,12 @@ maven-enforcer-plugin - - - enforce - none - - - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/sh - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/sh - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + + maven-resources-plugin diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java index dbae6c9bf6..b7c0043dd1 100644 --- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -174,8 +174,8 @@ public class ShellInterpreter extends KerberosInterpreter { return false; } - public void createSecureConfiguration() { - Properties properties = getProperty(); + public void createSecureConfiguration() throws InterpreterException { + Properties properties = getProperties(); CommandLine cmdLine = CommandLine.parse(shell); cmdLine.addArgument("-c", false); String kinitCommand = String.format("kinit -k -t %s %s", diff --git a/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index 56b3823b59..a0505692d5 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -21,12 +21,15 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.python.IPythonInterpreter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Map; import java.util.Properties; /** @@ -43,8 +46,9 @@ public class IPySparkInterpreter extends IPythonInterpreter { } @Override - public void open() { - property.setProperty("zeppelin.python", PySparkInterpreter.getPythonExec(property)); + public void open() throws InterpreterException { + setProperty("zeppelin.python", + PySparkInterpreter.getPythonExec(getProperties())); sparkInterpreter = getSparkInterpreter(); SparkConf conf = sparkInterpreter.getSparkContext().getConf(); // only set PYTHONPATH in local or yarn-client mode. @@ -57,7 +61,18 @@ public class IPySparkInterpreter extends IPythonInterpreter { super.open(); } - private SparkInterpreter getSparkInterpreter() { + @Override + protected Map setupIPythonEnv() throws IOException { + Map env = super.setupIPythonEnv(); + // set PYSPARK_PYTHON + SparkConf conf = sparkInterpreter.getSparkContext().getConf(); + if (conf.contains("spark.pyspark.python")) { + env.put("PYSPARK_PYTHON", conf.get("spark.pyspark.python")); + } + return env; + } + + private SparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; SparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index dd32059a2c..5df4ec6b01 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -86,11 +86,11 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand File scriptFile = File.createTempFile("zeppelin_pyspark-", ".py"); scriptPath = scriptFile.getAbsolutePath(); } catch (IOException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } } - private void createPythonScript() { + private void createPythonScript() throws InterpreterException { ClassLoader classLoader = getClass().getClassLoader(); File out = new File(scriptPath); @@ -112,10 +112,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } @Override - public void open() { + public void open() throws InterpreterException { // try IPySparkInterpreter first iPySparkInterpreter = getIPySparkInterpreter(); - if (property.getProperty("zeppelin.pyspark.useIPython", "true").equals("true") && + if (getProperty("zeppelin.pyspark.useIPython", "true").equals("true") && iPySparkInterpreter.checkIPythonPrerequisite()) { try { iPySparkInterpreter.open(); @@ -132,8 +132,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } iPySparkInterpreter = null; - - if (property.getProperty("zeppelin.pyspark.useIPython", "true").equals("true")) { + if (getProperty("zeppelin.pyspark.useIPython", "true").equals("true")) { // don't print it when it is in testing, just for easy output check in test. try { InterpreterContext.get().out.write(("IPython is not available, " + @@ -202,7 +201,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - private Map setupPySparkEnv() throws IOException { + private Map setupPySparkEnv() throws IOException, InterpreterException { Map env = EnvironmentUtils.getProcEnvironment(); // only set PYTHONPATH in local or yarn-client mode. @@ -229,6 +228,11 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } LOGGER.info("PYTHONPATH: " + env.get("PYTHONPATH")); + + // set PYSPARK_PYTHON + if (getSparkConf().contains("spark.pyspark.python")) { + env.put("PYSPARK_PYTHON", getSparkConf().get("spark.pyspark.python")); + } return env; } @@ -246,7 +250,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand return pythonExec; } - private void createGatewayServerAndStartScript() { + private void createGatewayServerAndStartScript() throws InterpreterException { // create python script createPythonScript(); @@ -255,7 +259,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand gatewayServer = new GatewayServer(this, port); gatewayServer.start(); - String pythonExec = getPythonExec(property); + String pythonExec = getPythonExec(getProperties()); LOGGER.info("pythonExec: " + pythonExec); CommandLine cmd = CommandLine.parse(pythonExec); cmd.addArgument(scriptPath, false); @@ -295,7 +299,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - private int findRandomOpenPortOnAllLocalInterfaces() { + private int findRandomOpenPortOnAllLocalInterfaces() throws InterpreterException { int port; try (ServerSocket socket = new ServerSocket(0);) { port = socket.getLocalPort(); @@ -394,7 +398,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { SparkInterpreter sparkInterpreter = getSparkInterpreter(); sparkInterpreter.populateSparkWebUrl(context); if (sparkInterpreter.isUnsupportedSparkVersion()) { @@ -500,7 +505,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { if (iPySparkInterpreter != null) { iPySparkInterpreter.cancel(context); return; @@ -520,7 +525,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { if (iPySparkInterpreter != null) { return iPySparkInterpreter.getProgress(context); } @@ -531,7 +536,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) throws InterpreterException { if (iPySparkInterpreter != null) { return iPySparkInterpreter.completion(buf, cursor, interpreterContext); } @@ -632,7 +637,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } - private SparkInterpreter getSparkInterpreter() { + private SparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; SparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); @@ -666,7 +671,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand return iPySpark; } - public SparkZeppelinContext getZeppelinContext() { + public SparkZeppelinContext getZeppelinContext() throws InterpreterException { SparkInterpreter sparkIntp = getSparkInterpreter(); if (sparkIntp != null) { return getSparkInterpreter().getZeppelinContext(); @@ -675,7 +680,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - public JavaSparkContext getJavaSparkContext() { + public JavaSparkContext getJavaSparkContext() throws InterpreterException { SparkInterpreter intp = getSparkInterpreter(); if (intp == null) { return null; @@ -684,7 +689,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - public Object getSparkSession() { + public Object getSparkSession() throws InterpreterException { SparkInterpreter intp = getSparkInterpreter(); if (intp == null) { return null; @@ -693,7 +698,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - public SparkConf getSparkConf() { + public SparkConf getSparkConf() throws InterpreterException { JavaSparkContext sc = getJavaSparkContext(); if (sc == null) { return null; @@ -702,7 +707,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand } } - public SQLContext getSQLContext() { + public SQLContext getSQLContext() throws InterpreterException { SparkInterpreter intp = getSparkInterpreter(); if (intp == null) { return null; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 18da0344f0..5a5110f9fe 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -353,7 +353,7 @@ public class SparkInterpreter extends Interpreter { public boolean isYarnMode() { String master = getProperty("master"); if (master == null) { - master = getProperty().getProperty("spark.master", "local[*]"); + master = getProperty("spark.master", "local[*]"); } return master.startsWith("yarn"); } @@ -376,7 +376,7 @@ public class SparkInterpreter extends Interpreter { } conf.set("spark.scheduler.mode", "FAIR"); - Properties intpProperty = getProperty(); + Properties intpProperty = getProperties(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); @@ -509,7 +509,7 @@ public class SparkInterpreter extends Interpreter { } conf.set("spark.scheduler.mode", "FAIR"); - Properties intpProperty = getProperty(); + Properties intpProperty = getProperties(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); @@ -543,19 +543,19 @@ public class SparkInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean( - property.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); + getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); // set properties and do login before creating any spark stuff for secured cluster if (isYarnMode()) { System.setProperty("SPARK_YARN_MODE", "true"); } - if (getProperty().containsKey("spark.yarn.keytab") && - getProperty().containsKey("spark.yarn.principal")) { + if (getProperties().containsKey("spark.yarn.keytab") && + getProperties().containsKey("spark.yarn.principal")) { try { - String keytab = getProperty().getProperty("spark.yarn.keytab"); - String principal = getProperty().getProperty("spark.yarn.principal"); + String keytab = getProperties().getProperty("spark.yarn.keytab"); + String principal = getProperties().getProperty("spark.yarn.principal"); UserGroupInformation.loginUserFromKeytab(principal, keytab); } catch (IOException e) { throw new RuntimeException("Can not pass kerberos authentication", e); @@ -963,7 +963,7 @@ public class SparkInterpreter extends Interpreter { sparkUrl = getSparkUIUrl(); Map infos = new java.util.HashMap<>(); infos.put("url", sparkUrl); - String uiEnabledProp = property.getProperty("spark.ui.enabled", "true"); + String uiEnabledProp = getProperty("spark.ui.enabled", "true"); java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean( uiEnabledProp.trim()); if (!uiEnabled) { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index ca52f790f6..1bdd4dc07b 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -54,7 +54,7 @@ public class SparkRInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { String rCmdPath = getProperty("zeppelin.R.cmd"); String sparkRLibPath; @@ -105,7 +105,8 @@ public class SparkRInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) { + public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) + throws InterpreterException { SparkInterpreter sparkInterpreter = getSparkInterpreter(); sparkInterpreter.populateSparkWebUrl(interpreterContext); @@ -220,7 +221,7 @@ public class SparkRInterpreter extends Interpreter { return new ArrayList<>(); } - private SparkInterpreter getSparkInterpreter() { + private SparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; SparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 134a65f39c..9709f9eea9 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -59,7 +59,7 @@ public class SparkSqlInterpreter extends Interpreter { this.maxResult = Integer.parseInt(getProperty(MAX_RESULTS)); } - private SparkInterpreter getSparkInterpreter() { + private SparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; SparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); @@ -86,7 +86,8 @@ public class SparkSqlInterpreter extends Interpreter { public void close() {} @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { SQLContext sqlc = null; SparkInterpreter sparkInterpreter = getSparkInterpreter(); @@ -134,7 +135,7 @@ public class SparkSqlInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { SparkInterpreter sparkInterpreter = getSparkInterpreter(); SQLContext sqlc = sparkInterpreter.getSQLContext(); SparkContext sc = sqlc.sparkContext(); @@ -149,7 +150,7 @@ public class SparkSqlInterpreter extends Interpreter { @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { SparkInterpreter sparkInterpreter = getSparkInterpreter(); return sparkInterpreter.getProgress(context); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java index 413c690711..09b8d440cc 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java @@ -79,7 +79,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext { } if (supportedClasses.isEmpty()) { - throw new InterpreterException("Can not load Dataset/DataFrame/SchemaRDD class"); + throw new RuntimeException("Can not load Dataset/DataFrame/SchemaRDD class"); } } @@ -112,7 +112,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext { } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | ClassCastException e) { sc.clearJobGroup(); - throw new InterpreterException(e); + throw new RuntimeException(e); } List columns = null; @@ -129,7 +129,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext { .asJava(); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } StringBuilder msg = new StringBuilder(); @@ -165,7 +165,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext { } } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } if (rows.length > maxResult) { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java index b46001aa0e..130d849975 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java @@ -117,7 +117,7 @@ public class ZeppelinR implements ExecuteResultHandler { File scriptFile = File.createTempFile("zeppelin_sparkr-", ".R"); scriptPath = scriptFile.getAbsolutePath(); } catch (IOException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } } @@ -125,7 +125,7 @@ public class ZeppelinR implements ExecuteResultHandler { * Start R repl * @throws IOException */ - public void open() throws IOException { + public void open() throws IOException, InterpreterException { createRScript(); zeppelinR.put(hashCode(), this); @@ -170,7 +170,7 @@ public class ZeppelinR implements ExecuteResultHandler { * @param expr * @return */ - public Object eval(String expr) { + public Object eval(String expr) throws InterpreterException { synchronized (this) { rRequestObject = new Request("eval", expr, null); return request(); @@ -182,7 +182,7 @@ public class ZeppelinR implements ExecuteResultHandler { * @param key * @param value */ - public void set(String key, Object value) { + public void set(String key, Object value) throws InterpreterException { synchronized (this) { rRequestObject = new Request("set", key, value); request(); @@ -194,7 +194,7 @@ public class ZeppelinR implements ExecuteResultHandler { * @param key * @return */ - public Object get(String key) { + public Object get(String key) throws InterpreterException { synchronized (this) { rRequestObject = new Request("get", key, null); return request(); @@ -206,7 +206,7 @@ public class ZeppelinR implements ExecuteResultHandler { * @param key * @return */ - public String getS0(String key) { + public String getS0(String key) throws InterpreterException { synchronized (this) { rRequestObject = new Request("getS", key, null); return (String) request(); @@ -217,7 +217,7 @@ public class ZeppelinR implements ExecuteResultHandler { * Send request to r repl and return response * @return responseValue */ - private Object request() throws RuntimeException { + private Object request() throws RuntimeException, InterpreterException { if (!rScriptRunning) { throw new RuntimeException("r repl is not running"); } @@ -332,7 +332,7 @@ public class ZeppelinR implements ExecuteResultHandler { /** * Create R script in tmp dir */ - private void createRScript() { + private void createRScript() throws InterpreterException { ClassLoader classLoader = getClass().getClassLoader(); File out = new File(scriptPath); diff --git a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index 3f7cf752d4..faf0473092 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -24,6 +24,7 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -55,7 +56,7 @@ public class IPySparkInterpreterTest { private InterpreterGroup intpGroup; @Before - public void setup() { + public void setup() throws InterpreterException { Properties p = new Properties(); p.setProperty("spark.master", "local[4]"); p.setProperty("master", "local[4]"); @@ -90,7 +91,7 @@ public class IPySparkInterpreterTest { } @Test - public void testBasics() throws InterruptedException, IOException { + public void testBasics() throws InterruptedException, IOException, InterpreterException { // all the ipython test should pass too. IPythonInterpreterTest.testInterpreter(iPySparkInterpreter); diff --git a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java b/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java index d6950372f0..692447e698 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java @@ -64,7 +64,7 @@ public class PySparkInterpreterMatplotlibTest { * normally handles this in real use cases. */ @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) throws InterpreterException { context.out.clear(); InterpreterResult result = super.interpret(st, context); List resultMessages = null; @@ -140,7 +140,7 @@ public class PySparkInterpreterMatplotlibTest { } @Test - public void dependenciesAreInstalled() { + public void dependenciesAreInstalled() throws InterpreterException { // matplotlib InterpreterResult ret = pyspark.interpret("import matplotlib", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); @@ -151,7 +151,7 @@ public class PySparkInterpreterMatplotlibTest { } @Test - public void showPlot() { + public void showPlot() throws InterpreterException { // Simple plot test InterpreterResult ret; ret = pyspark.interpret("import matplotlib.pyplot as plt", context); @@ -168,7 +168,7 @@ public class PySparkInterpreterMatplotlibTest { @Test // Test for when configuration is set to auto-close figures after show(). - public void testClose() { + public void testClose() throws InterpreterException { InterpreterResult ret; InterpreterResult ret1; InterpreterResult ret2; @@ -195,7 +195,7 @@ public class PySparkInterpreterMatplotlibTest { @Test // Test for when configuration is set to not auto-close figures after show(). - public void testNoClose() { + public void testNoClose() throws InterpreterException { InterpreterResult ret; InterpreterResult ret1; InterpreterResult ret2; @@ -222,7 +222,7 @@ public class PySparkInterpreterMatplotlibTest { @Test // Test angular mode - public void testAngular() { + public void testAngular() throws InterpreterException { InterpreterResult ret; ret = pyspark.interpret("import matplotlib.pyplot as plt", context); ret = pyspark.interpret("plt.close()", context); diff --git a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java index 7a4abd61d8..a95c5efb07 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java @@ -112,7 +112,7 @@ public class PySparkInterpreterTest { } @Test - public void testBasicIntp() { + public void testBasicIntp() throws InterpreterException { if (getSparkVersionNumber() > 11) { assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret("a = 1\n", context).code()); @@ -136,7 +136,7 @@ public class PySparkInterpreterTest { } @Test - public void testCompletion() { + public void testCompletion() throws InterpreterException { if (getSparkVersionNumber() > 11) { List completions = pySparkInterpreter.completion("sc.", "sc.".length(), null); assertTrue(completions.size() > 0); @@ -144,7 +144,7 @@ public class PySparkInterpreterTest { } @Test - public void testRedefinitionZeppelinContext() { + public void testRedefinitionZeppelinContext() throws InterpreterException { if (getSparkVersionNumber() > 11) { String redefinitionCode = "z = 1\n"; String restoreCode = "z = __zeppelin__\n"; @@ -162,7 +162,12 @@ public class PySparkInterpreterTest { @Override public void run() { String code = "import time\nwhile True:\n time.sleep(1)" ; - InterpreterResult ret = pySparkInterpreter.interpret(code, context); + InterpreterResult ret = null; + try { + ret = pySparkInterpreter.interpret(code, context); + } catch (InterpreterException e) { + e.printStackTrace(); + } assertNotNull(ret); Pattern expectedMessage = Pattern.compile("KeyboardInterrupt"); Matcher m = expectedMessage.matcher(ret.message().toString()); @@ -171,7 +176,7 @@ public class PySparkInterpreterTest { } @Test - public void testCancelIntp() throws InterruptedException { + public void testCancelIntp() throws InterruptedException, InterpreterException { if (getSparkVersionNumber() > 11) { assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret("a = 1\n", context).code()); diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index ece292b829..aaf13c321c 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -213,7 +213,7 @@ public class SparkInterpreterTest { } @Test - public void testSparkSql() throws IOException { + public void testSparkSql() throws IOException, InterpreterException { repl.interpret("case class Person(name:String, age:Int)\n", context); repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code()); @@ -243,7 +243,7 @@ public class SparkInterpreterTest { @Test public void emptyConfigurationVariablesOnlyForNonSparkProperties() { - Properties intpProperty = repl.getProperty(); + Properties intpProperty = repl.getProperties(); SparkConf sparkConf = repl.getSparkContext().getConf(); for (Object oKey : intpProperty.keySet()) { String key = (String) oKey; @@ -256,7 +256,7 @@ public class SparkInterpreterTest { } @Test - public void shareSingleSparkContext() throws InterruptedException, IOException { + public void shareSingleSparkContext() throws InterruptedException, IOException, InterpreterException { // create another SparkInterpreter SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties(tmpDir)); repl2.setInterpreterGroup(intpGroup); @@ -272,7 +272,7 @@ public class SparkInterpreterTest { } @Test - public void testEnableImplicitImport() throws IOException { + public void testEnableImplicitImport() throws IOException, InterpreterException { if (getSparkVersionNumber(repl) >= 13) { // Set option of importing implicits to "true", and initialize new Spark repl Properties p = getSparkTestProperties(tmpDir); @@ -289,7 +289,7 @@ public class SparkInterpreterTest { } @Test - public void testDisableImplicitImport() throws IOException { + public void testDisableImplicitImport() throws IOException, InterpreterException { if (getSparkVersionNumber(repl) >= 13) { // Set option of importing implicits to "false", and initialize new Spark repl // this test should return error status when creating DataFrame from sequence diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index ebb5e9a911..3e33f440b1 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -85,7 +85,7 @@ public class SparkSqlInterpreterTest { } @Test - public void test() { + public void test() throws InterpreterException { repl.interpret("case class Test(name:String, age:Int)", context); repl.interpret("val test = sc.parallelize(Seq(Test(\"moon\", 33), Test(\"jobs\", 51), Test(\"gates\", 51), Test(\"park\", 34)))", context); if (isDataFrameSupported()) { @@ -107,7 +107,7 @@ public class SparkSqlInterpreterTest { } @Test - public void testStruct() { + public void testStruct() throws InterpreterException { repl.interpret("case class Person(name:String, age:Int)", context); repl.interpret("case class People(group:String, person:Person)", context); repl.interpret( @@ -124,7 +124,7 @@ public class SparkSqlInterpreterTest { } @Test - public void test_null_value_in_row() { + public void test_null_value_in_row() throws InterpreterException { repl.interpret("import org.apache.spark.sql._", context); if (isDataFrameSupported()) { repl.interpret( @@ -162,7 +162,7 @@ public class SparkSqlInterpreterTest { } @Test - public void testMaxResults() { + public void testMaxResults() throws InterpreterException { repl.interpret("case class P(age:Int)", context); repl.interpret( "val gr = sc.parallelize(Seq(P(1),P(2),P(3),P(4),P(5),P(6),P(7),P(8),P(9),P(10),P(11)))", diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 3a6d0aac88..ab120f235f 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -217,6 +217,7 @@ The following components are provided under Apache License. (Apache 2.0) frontend-maven-plugin 1.3 (com.github.eirslett:frontend-maven-plugin:1.3 - https://github.com/eirslett/frontend-maven-plugin/blob/frontend-plugins-1.3/LICENSE (Apache 2.0) frontend-plugin-core 1.3 (com.github.eirslett:frontend-plugin-core) - https://github.com/eirslett/frontend-maven-plugin/blob/frontend-plugins-1.3/LICENSE (Apache 2.0) mongo-java-driver 3.4.1 (org.mongodb:mongo-java-driver:3.4.1) - https://github.com/mongodb/mongo-java-driver/blob/master/LICENSE.txt + (Apache 2.0) Neo4j Java Driver (https://github.com/neo4j/neo4j-java-driver) - https://github.com/neo4j/neo4j-java-driver/blob/1.4.3/LICENSE.txt ======================================================================== MIT licenses @@ -273,6 +274,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE (The MIT License) angular-viewport-watch 0.135 (https://github.com/wix/angular-viewport-watch) - https://github.com/wix/angular-viewport-watch/blob/master/LICENSE (The MIT License) ansi-up 2.0.2 (https://github.com/drudru/ansi_up) - https://github.com/drudru/ansi_up#license + (The MIT License) bcpkix-jdk15on 1.52 (org.bouncycastle:bcpkix-jdk15on:1.52 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html ======================================================================== BSD-style licenses diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 384b9d1c59..3bb50c732f 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -66,6 +66,11 @@ gson-extras + + commons-configuration + commons-configuration + + org.apache.commons commons-exec @@ -132,6 +137,12 @@ ${aether.version} + + org.bouncycastle + bcpkix-jdk15on + 1.52 + + org.apache.maven maven-aether-provider @@ -226,4 +237,79 @@ test + + + + + + + maven-enforcer-plugin + + + enforce + none + + + + + + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + ${basedir}/interpreter/${project.name} + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${basedir}/interpreter/${project.name} + false + false + true + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + maven-resources-plugin + + + copy-resources + package + + resources + + + ${basedir}/interpreter/{project.name} + + + + + + + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java similarity index 98% rename from zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 2dec19cbf5..3a82bc5898 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -443,6 +443,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getBoolean(ConfVars.ZEPPELIN_CREDENTIALS_PERSIST); } + public String getCredentialsEncryptKey() { + return getString(ConfVars.ZEPPELIN_CREDENTIALS_ENCRYPT_KEY); + } + public String getCredentialsPath() { return getRelativeDir(String.format("%s/credentials.json", getConfDir())); } @@ -623,7 +627,8 @@ public class ZeppelinConfiguration extends XMLConfiguration { + "org.apache.zeppelin.bigquery.BigQueryInterpreter," + "org.apache.zeppelin.beam.BeamInterpreter," + "org.apache.zeppelin.scio.ScioInterpreter," - + "org.apache.zeppelin.groovy.GroovyInterpreter" + + "org.apache.zeppelin.groovy.GroovyInterpreter," + + "org.apache.zeppelin.neo4j.Neo4jCypherInterpreter" ), ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), @@ -634,7 +639,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh," + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch," - + "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy"), + + "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy,neo4j"), ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"), @@ -680,6 +685,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true), ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true), + ZEPPELIN_CREDENTIALS_ENCRYPT_KEY("zeppelin.credentials.encryptKey", null), ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"), ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false), ZEPPELIN_SERVER_XFRAME_OPTIONS("zeppelin.server.xframe.options", "SAMEORIGIN"), @@ -687,8 +693,8 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_SERVER_STRICT_TRANSPORT("zeppelin.server.strict.transport", "max-age=631138519"), ZEPPELIN_SERVER_X_XSS_PROTECTION("zeppelin.server.xxss.protection", "1"), - ZEPPELIN_HDFS_KEYTAB("zeppelin.hdfs.keytab", ""), - ZEPPELIN_HDFS_PRINCIPAL("zeppelin.hdfs.principal", ""), + ZEPPELIN_SERVER_KERBEROS_KEYTAB("zeppelin.server.kerberos.keytab", ""), + ZEPPELIN_SERVER_KERBEROS_PRINCIPAL("zeppelin.server.kerberos.principal", ""), ZEPPELIN_INTERPRETER_CALLBACK_PORTRANGE("zeppelin.interpreter.callback.portRange", ":"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java index 12376f0dd7..9503962259 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/BaseZeppelinContext.java @@ -168,7 +168,7 @@ public abstract class BaseZeppelinContext { interpreterContext.out.write(o.toString()); } } catch (IOException e) { - throw new InterpreterException(e); + throw new RuntimeException(e); } } @@ -229,14 +229,14 @@ public abstract class BaseZeppelinContext { public void run(String noteId, String paragraphId, InterpreterContext context, boolean checkCurrentParagraph) { if (paragraphId.equals(context.getParagraphId()) && checkCurrentParagraph) { - throw new InterpreterException("Can not run current Paragraph"); + throw new RuntimeException("Can not run current Paragraph"); } List runners = getInterpreterContextRunner(noteId, paragraphId, context); if (runners.size() <= 0) { - throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size()); + throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size()); } for (InterpreterContextRunner r : runners) { @@ -255,7 +255,7 @@ public abstract class BaseZeppelinContext { List runners = getInterpreterContextRunner(noteId, context); if (runners.size() <= 0) { - throw new InterpreterException("Note " + noteId + " not found " + runners.size()); + throw new RuntimeException("Note " + noteId + " not found " + runners.size()); } for (InterpreterContextRunner r : runners) { @@ -346,12 +346,12 @@ public abstract class BaseZeppelinContext { boolean checkCurrentParagraph) { List runners = getInterpreterContextRunner(noteId, context); if (idx >= runners.size()) { - throw new InterpreterException("Index out of bound"); + throw new RuntimeException("Index out of bound"); } InterpreterContextRunner runner = runners.get(idx); if (runner.getParagraphId().equals(context.getParagraphId()) && checkCurrentParagraph) { - throw new InterpreterException("Can not run current Paragraph: " + runner.getParagraphId()); + throw new RuntimeException("Can not run current Paragraph: " + runner.getParagraphId()); } runner.run(); @@ -377,7 +377,7 @@ public abstract class BaseZeppelinContext { Integer idx = (Integer) idOrIdx; run(noteId, idx, context); } else { - throw new InterpreterException("Paragraph " + idOrIdx + " not found"); + throw new RuntimeException("Paragraph " + idOrIdx + " not found"); } } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java index a1dafd9721..f8afa45737 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ClassloaderInterpreter.java @@ -51,7 +51,8 @@ public class ClassloaderInterpreter } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { @@ -68,7 +69,7 @@ public class ClassloaderInterpreter @Override - public void open() { + public void open() throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { @@ -82,7 +83,7 @@ public class ClassloaderInterpreter } @Override - public void close() { + public void close() throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { @@ -96,7 +97,7 @@ public class ClassloaderInterpreter } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { @@ -110,13 +111,11 @@ public class ClassloaderInterpreter } @Override - public FormType getFormType() { + public FormType getFormType() throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { return intp.getFormType(); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -124,7 +123,7 @@ public class ClassloaderInterpreter } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { @@ -143,8 +142,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { return intp.getScheduler(); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -153,14 +150,12 @@ public class ClassloaderInterpreter @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) throws InterpreterException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { List completion = intp.completion(buf, cursor, interpreterContext); return completion; - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -174,8 +169,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { return intp.getClassName(); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -188,8 +181,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { intp.setInterpreterGroup(interpreterGroup); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -202,8 +193,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { return intp.getInterpreterGroup(); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -216,8 +205,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { intp.setClassloaderUrls(urls); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -230,8 +217,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { return intp.getClassloaderUrls(); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -239,13 +224,11 @@ public class ClassloaderInterpreter } @Override - public void setProperty(Properties property) { + public void setProperties(Properties properties) { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { - intp.setProperty(property); - } catch (Exception e) { - throw new InterpreterException(e); + intp.setProperties(properties); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -253,13 +236,11 @@ public class ClassloaderInterpreter } @Override - public Properties getProperty() { + public Properties getProperties() { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { - return intp.getProperty(); - } catch (Exception e) { - throw new InterpreterException(e); + return intp.getProperties(); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); @@ -272,8 +253,6 @@ public class ClassloaderInterpreter Thread.currentThread().setContextClassLoader(cl); try { return intp.getProperty(key); - } catch (Exception e) { - throw new InterpreterException(e); } finally { cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(oldcl); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java index 05599a01a3..386de4178c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java @@ -31,6 +31,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.reflect.FieldUtils; import org.apache.zeppelin.annotation.Experimental; import org.apache.zeppelin.annotation.ZeppelinApi; +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -55,20 +56,21 @@ public abstract class Interpreter { * open() is called only once */ @ZeppelinApi - public abstract void open(); + public abstract void open() throws InterpreterException; /** * Closes interpreter. You may want to free your resources up here. * close() is called only once */ @ZeppelinApi - public abstract void close(); + public abstract void close() throws InterpreterException; /** * Run precode if exists. */ @ZeppelinApi - public InterpreterResult executePrecode(InterpreterContext interpreterContext) { + public InterpreterResult executePrecode(InterpreterContext interpreterContext) + throws InterpreterException { String simpleName = this.getClass().getSimpleName(); String precode = getProperty(String.format("zeppelin.%s.precode", simpleName)); if (StringUtils.isNotBlank(precode)) { @@ -83,13 +85,15 @@ public abstract class Interpreter { * @param st statements to run */ @ZeppelinApi - public abstract InterpreterResult interpret(String st, InterpreterContext context); + public abstract InterpreterResult interpret(String st, + InterpreterContext context) + throws InterpreterException; /** * Optionally implement the canceling routine to abort interpret() method */ @ZeppelinApi - public abstract void cancel(InterpreterContext context); + public abstract void cancel(InterpreterContext context) throws InterpreterException; /** * Dynamic form handling @@ -99,7 +103,7 @@ public abstract class Interpreter { * FormType.NATIVE handles form in API */ @ZeppelinApi - public abstract FormType getFormType(); + public abstract FormType getFormType() throws InterpreterException; /** * get interpret() method running process in percentage. @@ -107,7 +111,7 @@ public abstract class Interpreter { * @return number between 0-100 */ @ZeppelinApi - public abstract int getProgress(InterpreterContext context); + public abstract int getProgress(InterpreterContext context) throws InterpreterException; /** * Get completion list based on cursor position. @@ -120,7 +124,7 @@ public abstract class Interpreter { */ @ZeppelinApi public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) throws InterpreterException { return null; } @@ -144,22 +148,22 @@ public abstract class Interpreter { public static Logger logger = LoggerFactory.getLogger(Interpreter.class); private InterpreterGroup interpreterGroup; private URL[] classloaderUrls; - protected Properties property; - private String userName; + protected Properties properties; + protected String userName; @ZeppelinApi - public Interpreter(Properties property) { - this.property = property; + public Interpreter(Properties properties) { + this.properties = properties; } - public void setProperty(Properties property) { - this.property = property; + public void setProperties(Properties properties) { + this.properties = properties; } @ZeppelinApi - public Properties getProperty() { + public Properties getProperties() { Properties p = new Properties(); - p.putAll(property); + p.putAll(properties); RegisteredInterpreter registeredInterpreter = Interpreter.findRegisteredInterpreterByClassName( getClassName()); @@ -183,11 +187,22 @@ public abstract class Interpreter { @ZeppelinApi public String getProperty(String key) { - logger.debug("key: {}, value: {}", key, getProperty().getProperty(key)); + logger.debug("key: {}, value: {}", key, getProperties().getProperty(key)); - return getProperty().getProperty(key); + return getProperties().getProperty(key); } + @ZeppelinApi + public String getProperty(String key, String defaultValue) { + logger.debug("key: {}, value: {}", key, getProperties().getProperty(key, defaultValue)); + + return getProperties().getProperty(key, defaultValue); + } + + @ZeppelinApi + public void setProperty(String key, String value) { + properties.setProperty(key, value); + } public String getClassName() { return this.getClass().getName(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java index ebd184ecfb..8b8a229765 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterException.java @@ -17,11 +17,12 @@ package org.apache.zeppelin.interpreter; + /** * Runtime Exception for interpreters. * */ -public class InterpreterException extends RuntimeException { +public class InterpreterException extends Exception { public InterpreterException(Throwable e) { super(e); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index 5428cdbb53..6acd601b87 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -26,7 +26,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Random; +import java.security.SecureRandom; import java.util.concurrent.ConcurrentHashMap; /** @@ -70,7 +70,7 @@ public class InterpreterGroup { } private static String generateId() { - return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt(); + return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new SecureRandom().nextInt(); } public String getId() { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 37a0d99c4a..e8a92255fc 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -28,7 +28,8 @@ public class InterpreterOption { public static final transient String SCOPED = "scoped"; public static final transient String ISOLATED = "isolated"; - boolean remote; + // always set it as true, keep this field just for backward compatibility + boolean remote = true; String host = null; int port = -1; @@ -77,14 +78,9 @@ public class InterpreterOption { } public InterpreterOption() { - this(false); } - public InterpreterOption(boolean remote) { - this(remote, SHARED, SHARED); - } - - public InterpreterOption(boolean remote, String perUser, String perNote) { + public InterpreterOption(String perUser, String perNote) { if (perUser == null) { throw new NullPointerException("perUser can not be null."); } @@ -92,7 +88,6 @@ public class InterpreterOption { throw new NullPointerException("perNote can not be null."); } - this.remote = remote; this.perUser = perUser; this.perNote = perNote; } @@ -112,14 +107,6 @@ public class InterpreterOption { return option; } - public boolean isRemote() { - return remote; - } - - public void setRemote(boolean remote) { - this.remote = remote; - } - public String getHost() { return host; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java index 96f88eeb5c..8e55c31202 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/LazyOpenInterpreter.java @@ -44,13 +44,13 @@ public class LazyOpenInterpreter } @Override - public void setProperty(Properties property) { - intp.setProperty(property); + public void setProperties(Properties properties) { + intp.setProperties(properties); } @Override - public Properties getProperty() { - return intp.getProperty(); + public Properties getProperties() { + return intp.getProperties(); } @Override @@ -59,7 +59,7 @@ public class LazyOpenInterpreter } @Override - public synchronized void open() { + public synchronized void open() throws InterpreterException { if (opened == true) { return; } @@ -73,12 +73,13 @@ public class LazyOpenInterpreter } @Override - public InterpreterResult executePrecode(InterpreterContext interpreterContext) { + public InterpreterResult executePrecode(InterpreterContext interpreterContext) + throws InterpreterException { return intp.executePrecode(interpreterContext); } @Override - public void close() { + public void close() throws InterpreterException { synchronized (intp) { if (opened == true) { intp.close(); @@ -94,7 +95,8 @@ public class LazyOpenInterpreter } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { open(); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); try { @@ -105,18 +107,18 @@ public class LazyOpenInterpreter } @Override - public void cancel(InterpreterContext context) { + public void cancel(InterpreterContext context) throws InterpreterException { open(); intp.cancel(context); } @Override - public FormType getFormType() { + public FormType getFormType() throws InterpreterException { return intp.getFormType(); } @Override - public int getProgress(InterpreterContext context) { + public int getProgress(InterpreterContext context) throws InterpreterException { if (opened) { return intp.getProgress(context); } else { @@ -131,7 +133,7 @@ public class LazyOpenInterpreter @Override public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { + InterpreterContext interpreterContext) throws InterpreterException { open(); List completion = intp.completion(buf, cursor, interpreterContext); return completion; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/graph/GraphResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/graph/GraphResult.java new file mode 100644 index 0000000000..df1b9a3ae6 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/graph/GraphResult.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.graph; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.tabledata.Node; +import org.apache.zeppelin.tabledata.Relationship; + +import com.google.gson.Gson; + +/** + * The intepreter result template for Networks + * + */ +public class GraphResult extends InterpreterResult { + + /** + * The Graph structure parsed from the front-end + * + */ + public static class Graph { + private Collection nodes; + + private Collection edges; + + /** + * The node types in the whole graph, and the related colors + * + */ + private Map labels; + + /** + * The relationship types in the whole graph + * + */ + private Set types; + + /** + * Is a directed graph + */ + private boolean directed; + + public Graph() {} + + public Graph(Collection nodes, Collection edges, + Map labels, Set types, boolean directed) { + super(); + this.setNodes(nodes); + this.setEdges(edges); + this.setLabels(labels); + this.setTypes(types); + this.setDirected(directed); + } + + public Collection getNodes() { + return nodes; + } + + public void setNodes(Collection nodes) { + this.nodes = nodes; + } + + public Collection getEdges() { + return edges; + } + + public void setEdges(Collection edges) { + this.edges = edges; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public Set getTypes() { + return types; + } + + public void setTypes(Set types) { + this.types = types; + } + + public boolean isDirected() { + return directed; + } + + public void setDirected(boolean directed) { + this.directed = directed; + } + + } + + private static final Gson gson = new Gson(); + + public GraphResult(Code code, Graph graphObject) { + super(code, Type.NETWORK, gson.toJson(graphObject)); + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java new file mode 100644 index 0000000000..b991079fec --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +/** + * Interface to InterpreterClient which is created by InterpreterLauncher. This is the component + * that is used to for the communication fromzeppelin-server process to zeppelin interpreter process + */ +public interface InterpreterClient { + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java new file mode 100644 index 0000000000..db8f8dd2fa --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterRunner; + +import java.util.Properties; + +/** + * Context class for Interpreter Launch + */ +public class InterpreterLaunchContext { + + private Properties properties; + private InterpreterOption option; + private InterpreterRunner runner; + private String interpreterGroupId; + private String interpreterGroupName; + + public InterpreterLaunchContext(Properties properties, + InterpreterOption option, + InterpreterRunner runner, + String interpreterGroupId, + String interpreterGroupName) { + this.properties = properties; + this.option = option; + this.runner = runner; + this.interpreterGroupId = interpreterGroupId; + this.interpreterGroupName = interpreterGroupName; + } + + public Properties getProperties() { + return properties; + } + + public InterpreterOption getOption() { + return option; + } + + public InterpreterRunner getRunner() { + return runner; + } + + public String getInterpreterGroupId() { + return interpreterGroupId; + } + + public String getInterpreterGroupName() { + return interpreterGroupName; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java new file mode 100644 index 0000000000..5d0acf3515 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; + +import java.io.IOException; +import java.util.Properties; + +/** + * Component to Launch interpreter process. + */ +public abstract class InterpreterLauncher { + + protected ZeppelinConfiguration zConf; + protected Properties properties; + + public InterpreterLauncher(ZeppelinConfiguration zConf) { + this.zConf = zConf; + } + + public abstract InterpreterClient launch(InterpreterLaunchContext context) throws IOException; +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java index 8d16ec52b1..74b8db6d9d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterContextRunner.java @@ -33,6 +33,6 @@ public class RemoteInterpreterContextRunner extends InterpreterContextRunner { public void run() { // this class should be used only for gson deserialize abstract class // code should not reach here - throw new InterpreterException("Assert"); + throw new RuntimeException("Assert"); } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 7f476e8026..cb0488cc07 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -207,7 +207,11 @@ public class RemoteInterpreterServer if (interpreterGroup != null) { for (List session : interpreterGroup.values()) { for (Interpreter interpreter : session) { - interpreter.close(); + try { + interpreter.close(); + } catch (InterpreterException e) { + logger.warn("Fail to close interpreter", e); + } } } } @@ -356,7 +360,11 @@ public class RemoteInterpreterServer public void open(String sessionId, String className) throws TException { logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId)); Interpreter intp = getInterpreter(sessionId, className); - intp.open(); + try { + intp.open(); + } catch (InterpreterException e) { + throw new TException("Fail to open interpreter", e); + } } @Override @@ -388,7 +396,11 @@ public class RemoteInterpreterServer while (it.hasNext()) { Interpreter inp = it.next(); if (inp.getClassName().equals(className)) { - inp.close(); + try { + inp.close(); + } catch (InterpreterException e) { + logger.warn("Fail to close interpreter", e); + } it.remove(); break; } @@ -655,7 +667,11 @@ public class RemoteInterpreterServer if (job != null) { job.setStatus(Status.ABORT); } else { - intp.cancel(convert(interpreterContext, null)); + try { + intp.cancel(convert(interpreterContext, null)); + } catch (InterpreterException e) { + throw new TException("Fail to cancel", e); + } } } @@ -672,7 +688,11 @@ public class RemoteInterpreterServer throw new TException("No interpreter {} existed for session {}".format( className, sessionId)); } - return intp.getProgress(convert(interpreterContext, null)); + try { + return intp.getProgress(convert(interpreterContext, null)); + } catch (InterpreterException e) { + throw new TException("Fail to getProgress", e); + } } } @@ -680,7 +700,11 @@ public class RemoteInterpreterServer @Override public String getFormType(String sessionId, String className) throws TException { Interpreter intp = getInterpreter(sessionId, className); - return intp.getFormType().toString(); + try { + return intp.getFormType().toString(); + } catch (InterpreterException e) { + throw new TException(e); + } } @Override @@ -688,8 +712,11 @@ public class RemoteInterpreterServer String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext) throws TException { Interpreter intp = getInterpreter(sessionId, className); - List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null)); - return completion; + try { + return intp.completion(buf, cursor, convert(remoteInterpreterContext, null)); + } catch (InterpreterException e) { + throw new TException("Fail to get completion", e); + } } private InterpreterContext convert(RemoteInterpreterContext ric) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/GraphEntity.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/GraphEntity.java new file mode 100644 index 0000000000..320b144726 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/GraphEntity.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.tabledata; + +import java.util.Map; + +/** + * The base network entity + * + */ +public abstract class GraphEntity { + + private long id; + + /** + * The data of the entity + * + */ + private Map data; + + /** + * The primary type of the entity + */ + private String label; + + public GraphEntity() {} + + public GraphEntity(long id, Map data, String label) { + super(); + this.setId(id); + this.setData(data); + this.setLabel(label); + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Node.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Node.java new file mode 100644 index 0000000000..2efabc4089 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Node.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.tabledata; + +import java.util.Map; +import java.util.Set; + +/** + * The Zeppelin Node Entity + * + */ +public class Node extends GraphEntity { + + /** + * The labels (types) attached to a node + */ + private Set labels; + + public Node() {} + + + public Node(long id, Map data, Set labels) { + super(id, data, labels.iterator().next()); + } + + public Set getLabels() { + return labels; + } + + public void setLabels(Set labels) { + this.labels = labels; + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Relationship.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Relationship.java new file mode 100644 index 0000000000..aa8ddb7854 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/Relationship.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.tabledata; + +import java.util.Map; + +/** + * The Zeppelin Relationship entity + * + */ +public class Relationship extends GraphEntity { + + /** + * Source node ID + */ + private long source; + + /** + * End node ID + */ + private long target; + + public Relationship() {} + + public Relationship(long id, Map data, long source, + long target, String label) { + super(id, data, label); + this.setSource(source); + this.setTarget(target); + } + + public long getSource() { + return source; + } + + public void setSource(long startNodeId) { + this.source = startNodeId; + } + + public long getTarget() { + return target; + } + + public void setTarget(long endNodeId) { + this.target = endNodeId; + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Credentials.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Credentials.java index e80a89f0e7..d345e3c507 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Credentials.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Credentials.java @@ -47,7 +47,21 @@ public class Credentials { private Boolean credentialsPersist = true; File credentialsFile; - public Credentials(Boolean credentialsPersist, String credentialsPath) { + private Encryptor encryptor; + + /** + * Wrapper fro user credentials. It can load credentials from a file if credentialsPath is + * supplied, and will encrypt the file if an encryptKey is supplied. + * + * @param credentialsPersist + * @param credentialsPath + * @param encryptKey + */ + public Credentials(Boolean credentialsPersist, String credentialsPath, String encryptKey) { + if (encryptKey != null) { + this.encryptor = new Encryptor(encryptKey); + } + this.credentialsPersist = credentialsPersist; if (credentialsPath != null) { credentialsFile = new File(credentialsPath); @@ -119,6 +133,11 @@ public class Credentials { fis.close(); String json = sb.toString(); + + if (encryptor != null) { + json = encryptor.decrypt(json); + } + CredentialsInfoSaving info = CredentialsInfoSaving.fromJson(json); this.credentialsMap = info.credentialsMap; } catch (IOException e) { @@ -146,6 +165,11 @@ public class Credentials { FileOutputStream fos = new FileOutputStream(credentialsFile, false); OutputStreamWriter out = new OutputStreamWriter(fos); + + if (encryptor != null) { + jsonString = encryptor.encrypt(jsonString); + } + out.append(jsonString); out.close(); fos.close(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Encryptor.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Encryptor.java new file mode 100644 index 0000000000..cf04b372bd --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/Encryptor.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.user; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.Arrays; + +import org.bouncycastle.crypto.BufferedBlockCipher; +import org.bouncycastle.crypto.InvalidCipherTextException; +import org.bouncycastle.crypto.engines.AESEngine; +import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher; +import org.bouncycastle.crypto.paddings.ZeroBytePadding; +import org.bouncycastle.crypto.params.KeyParameter; +import org.bouncycastle.util.encoders.Base64; + +/** + * Encrypt/decrypt arrays of bytes! + */ +public class Encryptor { + private final BufferedBlockCipher encryptCipher; + private final BufferedBlockCipher decryptCipher; + + public Encryptor(String encryptKey) { + encryptCipher = new PaddedBufferedBlockCipher(new AESEngine(), new ZeroBytePadding()); + encryptCipher.init(true, new KeyParameter(encryptKey.getBytes())); + + decryptCipher = new PaddedBufferedBlockCipher(new AESEngine(), new ZeroBytePadding()); + decryptCipher.init(false, new KeyParameter(encryptKey.getBytes())); + } + + + public String encrypt(String inputString) throws IOException { + byte[] input = inputString.getBytes(); + byte[] result = new byte[encryptCipher.getOutputSize(input.length)]; + int size = encryptCipher.processBytes(input, 0, input.length, result, 0); + + try { + size += encryptCipher.doFinal(result, size); + + byte[] out = new byte[size]; + System.arraycopy(result, 0, out, 0, size); + return new String(Base64.encode(out)); + } catch (InvalidCipherTextException e) { + throw new IOException("Cannot encrypt: " + e.getMessage(), e); + } + } + + public String decrypt(String base64Input) throws IOException { + byte[] input = Base64.decode(base64Input); + byte[] result = new byte[decryptCipher.getOutputSize(input.length)]; + int size = decryptCipher.processBytes(input, 0, input.length, result, 0); + + try { + size += decryptCipher.doFinal(result, size); + + byte[] out = new byte[size]; + System.arraycopy(result, 0, out, 0, size); + return new String(out); + } catch (InvalidCipherTextException e) { + throw new IOException("Cannot decrypt: " + e.getMessage(), e); + } + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java index 14c03a11cf..052aaefd21 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java @@ -20,7 +20,7 @@ package org.apache.zeppelin.util; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; -import java.util.Random; +import java.security.SecureRandom; /** * Generate Tiny ID. @@ -71,6 +71,6 @@ public class IdHashes { } public static String generateId() { - return encode(System.currentTimeMillis() + new Random().nextInt()); + return encode(System.currentTimeMillis() + new SecureRandom().nextInt()); } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java index 31c92256d8..d341b5864d 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java @@ -33,8 +33,8 @@ public class InterpreterTest { p.put("p1", "v1"); Interpreter intp = new DummyInterpreter(p); - assertEquals(1, intp.getProperty().size()); - assertEquals("v1", intp.getProperty().get("p1")); + assertEquals(1, intp.getProperties().size()); + assertEquals("v1", intp.getProperties().get("p1")); assertEquals("v1", intp.getProperty("p1")); } @@ -45,10 +45,10 @@ public class InterpreterTest { Interpreter intp = new DummyInterpreter(p); Properties overriddenProperty = new Properties(); overriddenProperty.put("p1", "v2"); - intp.setProperty(overriddenProperty); + intp.setProperties(overriddenProperty); - assertEquals(1, intp.getProperty().size()); - assertEquals("v2", intp.getProperty().get("p1")); + assertEquals(1, intp.getProperties().size()); + assertEquals("v2", intp.getProperties().get("p1")); assertEquals("v2", intp.getProperty("p1")); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java index 26e835f56d..8e325f2b5b 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java @@ -28,7 +28,7 @@ public class LazyOpenInterpreterTest { Interpreter interpreter = mock(Interpreter.class); @Test - public void isOpenTest() { + public void isOpenTest() throws InterpreterException { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); when(interpreter.interpret(any(String.class), any(InterpreterContext.class))).thenReturn(interpreterResult); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/CredentialsTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/CredentialsTest.java index 259516f976..4516bea845 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/CredentialsTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/CredentialsTest.java @@ -27,7 +27,7 @@ public class CredentialsTest { @Test public void testDefaultProperty() throws IOException { - Credentials credentials = new Credentials(false, null); + Credentials credentials = new Credentials(false, null, null); UserCredentials userCredentials = new UserCredentials(); UsernamePassword up1 = new UsernamePassword("user2", "password"); userCredentials.putUsernamePassword("hive(vertica)", up1); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/EncryptorTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/EncryptorTest.java new file mode 100644 index 0000000000..9950be6a63 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/user/EncryptorTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.user; + +import java.io.IOException; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class EncryptorTest { + + @Test + public void testEncryption() throws IOException { + Encryptor encryptor = new Encryptor("foobar1234567890"); + + String input = "test"; + + String encrypted = encryptor.encrypt(input); + assertNotEquals(input, encrypted); + + String decrypted = encryptor.decrypt(encrypted); + assertEquals(input, decrypted); + } +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index c1dba5c417..e2a10e62b3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -123,7 +123,7 @@ public class InterpreterRestApi { request.getOption(), request.getProperties()); logger.info("new setting created with {}", interpreterSetting.getId()); return new JsonResponse<>(Status.OK, "", interpreterSetting).build(); - } catch (InterpreterException | IOException e) { + } catch (IOException e) { logger.error("Exception in InterpreterRestApi while creating ", e); return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) .build(); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index c170a09901..42234fe29a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -41,10 +41,7 @@ import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.rest.exception.BadRequestException; import org.apache.zeppelin.rest.exception.NotFoundException; import org.apache.zeppelin.rest.exception.ForbiddenException; -import org.apache.zeppelin.rest.message.CronRequest; -import org.apache.zeppelin.rest.message.NewNoteRequest; -import org.apache.zeppelin.rest.message.NewParagraphRequest; -import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest; +import org.apache.zeppelin.rest.message.*; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.socket.NotebookServer; @@ -496,6 +493,37 @@ public class NotebookRestApi { return new JsonResponse<>(Status.OK, "", p).build(); } + /** + * Update paragraph + * + * @param message json containing the "text" and optionally the "title" of the paragraph, e.g. + * {"text" : "updated text", "title" : "Updated title" } + * + */ + @PUT + @Path("{noteId}/paragraph/{paragraphId}") + @ZeppelinApi + public Response updateParagraph(@PathParam("noteId") String noteId, + @PathParam("paragraphId") String paragraphId, + String message) throws IOException { + String user = SecurityUtils.getPrincipal(); + LOG.info("{} will update paragraph {} {}", user, noteId, paragraphId); + + Note note = notebook.getNote(noteId); + checkIfNoteIsNotNull(note); + checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph"); + Paragraph p = note.getParagraph(paragraphId); + checkIfParagraphIsNotNull(p); + + UpdateParagraphRequest updatedParagraph = gson.fromJson(message, UpdateParagraphRequest.class); + p.setText(updatedParagraph.getText()); + if (updatedParagraph.getTitle() != null) { p.setTitle(updatedParagraph.getTitle()); } + AuthenticationInfo subject = new AuthenticationInfo(user); + note.persist(subject); + notebookServer.broadcastParagraph(note, p); + return new JsonResponse<>(Status.OK, "").build(); + } + @PUT @Path("{noteId}/paragraph/{paragraphId}/config") @ZeppelinApi @@ -514,7 +542,6 @@ public class NotebookRestApi { configureParagraph(p, newConfig, user); AuthenticationInfo subject = new AuthenticationInfo(user); note.persist(subject); - return new JsonResponse<>(Status.OK, "", p).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateParagraphRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateParagraphRequest.java new file mode 100644 index 0000000000..3b4a6f68cc --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateParagraphRequest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest.message; + +/** + * UpdateParagraphRequest + */ +public class UpdateParagraphRequest { + String title; + String text; + + public UpdateParagraphRequest() { + + } + + public String getTitle() { + return title; + } + + public String getText() { + return text; + } + + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index c103eeb93a..f27bfbe27e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -132,7 +132,10 @@ public class ZeppelinServer extends Application { this.notebookRepo = new NotebookRepoSync(conf); this.noteSearchService = new LuceneSearch(); this.notebookAuthorization = NotebookAuthorization.init(conf); - this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); + this.credentials = new Credentials( + conf.credentialsPersist(), + conf.getCredentialsPath(), + conf.getCredentialsEncryptKey()); notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer, noteSearchService, notebookAuthorization, credentials); @@ -152,7 +155,7 @@ public class ZeppelinServer extends Application { } catch (Exception e) { LOG.error(e.getMessage(), e); } - + // to update notebook from application event from remote process. heliumApplicationFactory.setNotebook(notebook); // to update fire websocket event on application event. diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 5093cb838f..2a64665b0c 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Sets; import org.apache.commons.httpclient.methods.DeleteMethod; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; @@ -651,6 +652,43 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi { ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } + @Test + public void testUpdateParagraph() throws IOException { + Note note = ZeppelinServer.notebook.createNote(anonymous); + + String jsonRequest = "{\"title\": \"title1\", \"text\": \"text1\"}"; + PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest); + Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() {}.getType()); + post.releaseConnection(); + + String newParagraphId = (String) resp.get("body"); + Paragraph newParagraph = ZeppelinServer.notebook.getNote(note.getId()).getParagraph(newParagraphId); + + assertEquals("title1", newParagraph.getTitle()); + assertEquals("text1", newParagraph.getText()); + + String updateRequest = "{\"text\": \"updated text\"}"; + PutMethod put = httpPut("/notebook/" + note.getId() + "/paragraph/" + newParagraphId, updateRequest); + assertThat("Test update method:", put, isAllowed()); + put.releaseConnection(); + + Paragraph updatedParagraph = ZeppelinServer.notebook.getNote(note.getId()).getParagraph(newParagraphId); + + assertEquals("title1", updatedParagraph.getTitle()); + assertEquals("updated text", updatedParagraph.getText()); + + String updateBothRequest = "{\"title\": \"updated title\", \"text\" : \"updated text 2\" }"; + PutMethod updatePut = httpPut("/notebook/" + note.getId() + "/paragraph/" + newParagraphId, updateBothRequest); + updatePut.releaseConnection(); + + Paragraph updatedBothParagraph = ZeppelinServer.notebook.getNote(note.getId()).getParagraph(newParagraphId); + + assertEquals("updated title", updatedBothParagraph.getTitle()); + assertEquals("updated text 2", updatedBothParagraph.getText()); + + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + @Test public void testGetParagraph() throws IOException { Note note = ZeppelinServer.notebook.createNote(anonymous); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 3e464498ad..2fa584b9c4 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; @@ -171,7 +172,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { } @Test - public void sparkRTest() throws IOException { + public void sparkRTest() throws IOException, InterpreterException { // create new note Note note = ZeppelinServer.notebook.createNote(anonymous); int sparkVersion = getSparkVersionNumber(note); @@ -426,7 +427,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { } @Test - public void pySparkDepLoaderTest() throws IOException { + public void pySparkDepLoaderTest() throws IOException, InterpreterException { // create new note Note note = ZeppelinServer.notebook.createNote(anonymous); int sparkVersionNumber = getSparkVersionNumber(note); diff --git a/zeppelin-web/e2e/searchBlock.spec.js b/zeppelin-web/e2e/searchBlock.spec.js index 4a0ea48ca3..570673b838 100644 --- a/zeppelin-web/e2e/searchBlock.spec.js +++ b/zeppelin-web/e2e/searchBlock.spec.js @@ -99,7 +99,7 @@ describe('Search block e2e Test', function() { waitVisibility(element(by.repeater('currentParagraph in note.paragraphs'))) browser.switchTo().activeElement().sendKeys(testData.textInFirstP) let addBelow = element( - by.xpath('//div[@class="new-paragraph" and @ng-click="insertNew(\'below\');"]')) + by.xpath('//div[@class="new-paragraph last-paragraph" and @ng-click="insertNew(\'below\');"]')) clickAndWait(addBelow) browser.switchTo().activeElement().sendKeys(testData.textInSecondP) } @@ -123,7 +123,6 @@ describe('Search block e2e Test', function() { } /*Tests*/ - it('shortcut works', function() { waitVisibility(element(by.repeater('currentParagraph in note.paragraphs'))) openSearchBoxByShortcut() diff --git a/zeppelin-web/webpack.config.js b/zeppelin-web/webpack.config.js index 6fba4b1a30..3c349240ca 100644 --- a/zeppelin-web/webpack.config.js +++ b/zeppelin-web/webpack.config.js @@ -120,6 +120,7 @@ module.exports = function makeWebpackConfig () { * Reference: http://webpack.github.io/docs/configuration.html#devtool * Type of sourcemap to use per build type */ + config.devtool = 'eval-source-map'; if (isTest) { config.devtool = 'inline-source-map'; } else if (isProd) { @@ -127,7 +128,6 @@ module.exports = function makeWebpackConfig () { } else { config.devtool = 'eval-source-map'; } - config.devtool = 'source-map'; /** * Loaders @@ -196,14 +196,6 @@ module.exports = function makeWebpackConfig () { } ]}) }], - postLoaders: [ - { - // COVERAGE - test: /\.js$/, - exclude: /(node_modules|bower_components|\.test\.js)/, - loader: 'istanbul-instrumenter' - } - ] }; /** @@ -250,6 +242,17 @@ module.exports = function makeWebpackConfig () { }) ) } + + if (isTest) { + config.module.postLoaders = [ + { + // COVERAGE + test: /\.js$/, + exclude: /(node_modules|bower_components|\.test\.js)/, + loader: 'istanbul-instrumenter' + } + ] + } // Add build specific plugins if (isProd) { diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index c67df6b44a..d1a227052c 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -70,11 +70,6 @@ slf4j-log4j12 - - commons-configuration - commons-configuration - - commons-io commons-io diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index f020919442..7233239838 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -77,7 +77,7 @@ public class InterpreterFactory { return interpreter; } } - throw new InterpreterException(replName + " interpreter not found"); + return null; } else { // first assume replName is 'name' of interpreter. ('groupName' is ommitted) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index d7593d54e9..3d9c2c348a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -63,11 +63,6 @@ public class InterpreterInfoSaving implements JsonSerializable { if (infoSaving != null && infoSaving.interpreterSettings != null) { for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { - // Always use separate interpreter process - // While we decided to turn this feature on always (without providing - // enable/disable option on GUI). - // previously created setting should turn this feature on here. - interpreterSetting.getOption().setRemote(true); interpreterSetting.convertPermissionsFromUsersToOwners( jsonObject.getAsJsonObject("interpreterSettings") .getAsJsonObject(interpreterSetting.getId())); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 3a808c51d1..a82d5bf0e7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.interpreter; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; @@ -34,19 +33,22 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.launcher.InterpreterLaunchContext; +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; +import org.apache.zeppelin.interpreter.launcher.ShellScriptLauncher; +import org.apache.zeppelin.interpreter.launcher.SparkInterpreterLauncher; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; -import java.io.FilenameFilter; +import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URL; @@ -58,7 +60,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -105,7 +106,7 @@ public class InterpreterSetting { private List interpreterInfos; private List dependencies = new ArrayList<>(); - private InterpreterOption option = new InterpreterOption(true); + private InterpreterOption option = new InterpreterOption(); @SerializedName("runner") private InterpreterRunner interpreterRunner; @@ -132,8 +133,10 @@ public class InterpreterSetting { private transient ZeppelinConfiguration conf = new ZeppelinConfiguration(); - private transient Map cleanCl = - Collections.synchronizedMap(new HashMap()); + // TODO(zjffdu) ShellScriptLauncher is the only launcher implemention for now. It could be other + // launcher in future when we have other launcher implementation. e.g. third party launcher + // service like livy + private transient InterpreterLauncher launcher; /////////////////////////////////////////////////////////////////////////////////////////// @@ -245,6 +248,7 @@ public class InterpreterSetting { } void postProcessing() { +// createLauncher(); this.status = Status.READY; } @@ -268,6 +272,14 @@ public class InterpreterSetting { this.conf = o.getConf(); } + private void createLauncher() { + if (group.equals("spark")) { + this.launcher = new SparkInterpreterLauncher(this.conf); + } else { + this.launcher = new ShellScriptLauncher(this.conf); + } + } + public AngularObjectRegistryListener getAngularObjectRegistryListener() { return angularObjectRegistryListener; } @@ -616,13 +628,8 @@ public class InterpreterSetting { List interpreterInfos = getInterpreterInfos(); for (InterpreterInfo info : interpreterInfos) { Interpreter interpreter = null; - if (option.isRemote()) { - interpreter = new RemoteInterpreter(getJavaProperties(), sessionId, - info.getClassName(), user); - } else { - interpreter = createLocalInterpreter(info.getClassName()); - } - + interpreter = new RemoteInterpreter(getJavaProperties(), sessionId, + info.getClassName(), user); if (info.isDefaultInterpreter()) { interpreters.add(0, interpreter); } else { @@ -634,210 +641,16 @@ public class InterpreterSetting { return interpreters; } - // Create Interpreter in ZeppelinServer for non-remote mode - private Interpreter createLocalInterpreter(String className) - throws InterpreterException { - LOGGER.info("Create Local Interpreter {} from {}", className, interpreterDir); - - ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); - try { - - URLClassLoader ccl = cleanCl.get(interpreterDir); - if (ccl == null) { - // classloader fallback - ccl = URLClassLoader.newInstance(new URL[]{}, oldcl); - } - - boolean separateCL = true; - try { // check if server's classloader has driver already. - Class cls = this.getClass().forName(className); - if (cls != null) { - separateCL = false; - } - } catch (Exception e) { - LOGGER.error("exception checking server classloader driver", e); - } - - URLClassLoader cl; - - if (separateCL == true) { - cl = URLClassLoader.newInstance(new URL[]{}, ccl); - } else { - cl = ccl; - } - Thread.currentThread().setContextClassLoader(cl); - - Class replClass = (Class) cl.loadClass(className); - Constructor constructor = - replClass.getConstructor(new Class[]{Properties.class}); - Interpreter repl = constructor.newInstance(getJavaProperties()); - repl.setClassloaderUrls(ccl.getURLs()); - LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl)); - return intp; - } catch (SecurityException e) { - throw new InterpreterException(e); - } catch (NoSuchMethodException e) { - throw new InterpreterException(e); - } catch (IllegalArgumentException e) { - throw new InterpreterException(e); - } catch (InstantiationException e) { - throw new InterpreterException(e); - } catch (IllegalAccessException e) { - throw new InterpreterException(e); - } catch (InvocationTargetException e) { - throw new InterpreterException(e); - } catch (ClassNotFoundException e) { - throw new InterpreterException(e); - } finally { - Thread.currentThread().setContextClassLoader(oldcl); - } - } - - RemoteInterpreterProcess createInterpreterProcess() { - RemoteInterpreterProcess remoteInterpreterProcess = null; - int connectTimeout = - conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); - String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + id; - if (option.isExistingProcess()) { - // TODO(zjffdu) remove the existing process approach seems no one is using this. - // use the existing process - remoteInterpreterProcess = new RemoteInterpreterRunningProcess( - connectTimeout, - remoteInterpreterProcessListener, - appEventListener, - option.getHost(), - option.getPort()); - } else { - // create new remote process - remoteInterpreterProcess = new RemoteInterpreterManagedProcess( - interpreterRunner != null ? interpreterRunner.getPath() : - conf.getInterpreterRemoteRunnerPath(), conf.getCallbackPortRange(), - interpreterDir, localRepoPath, - getEnvFromInterpreterProperty(), connectTimeout, - remoteInterpreterProcessListener, appEventListener, group); - } - return remoteInterpreterProcess; - } - - private boolean isSparkConf(String key, String value) { - return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value); - } - - private Map getEnvFromInterpreterProperty() { - Map env = new HashMap(); - Properties javaProperties = getJavaProperties(); - Properties sparkProperties = new Properties(); - String sparkMaster = getSparkMaster(); - for (String key : javaProperties.stringPropertyNames()) { - if (RemoteInterpreterUtils.isEnvString(key)) { - env.put(key, javaProperties.getProperty(key)); - } - if (isSparkConf(key, javaProperties.getProperty(key))) { - sparkProperties.setProperty(key, toShellFormat(javaProperties.getProperty(key))); - } - } - - setupPropertiesForPySpark(sparkProperties); - setupPropertiesForSparkR(sparkProperties, System.getenv("SPARK_HOME")); - if (isYarnMode() && getDeployMode().equals("cluster")) { - env.put("SPARK_YARN_CLUSTER", "true"); - } - - StringBuilder sparkConfBuilder = new StringBuilder(); - if (sparkMaster != null) { - sparkConfBuilder.append(" --master " + sparkMaster); - } - if (isYarnMode() && getDeployMode().equals("cluster")) { - sparkConfBuilder.append(" --files " + conf.getConfDir() + "/log4j_yarn_cluster.properties"); - } - for (String name : sparkProperties.stringPropertyNames()) { - sparkConfBuilder.append(" --conf " + name + "=" + sparkProperties.getProperty(name)); - } - - env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString()); - LOGGER.debug("getEnvFromInterpreterProperty: " + env); - return env; - } - - private void setupPropertiesForPySpark(Properties sparkProperties) { - if (isYarnMode()) { - sparkProperties.setProperty("spark.yarn.isPython", "true"); - } - } - - private void mergeSparkProperty(Properties sparkProperties, String propertyName, - String propertyValue) { - if (sparkProperties.containsKey(propertyName)) { - String oldPropertyValue = sparkProperties.getProperty(propertyName); - sparkProperties.setProperty(propertyName, oldPropertyValue + "," + propertyValue); - } else { - sparkProperties.setProperty(propertyName, propertyValue); - } - } - - private void setupPropertiesForSparkR(Properties sparkProperties, - String sparkHome) { - File sparkRBasePath = null; - if (sparkHome == null) { - if (!getSparkMaster().startsWith("local")) { - throw new RuntimeException("SPARK_HOME is not specified for non-local mode"); - } - String zeppelinHome = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME); - sparkRBasePath = new File(zeppelinHome, - "interpreter" + File.separator + "spark" + File.separator + "R"); - } else { - sparkRBasePath = new File(sparkHome, "R" + File.separator + "lib"); - } - - File sparkRPath = new File(sparkRBasePath, "sparkr.zip"); - if (sparkRPath.exists() && sparkRPath.isFile()) { - mergeSparkProperty(sparkProperties, "spark.yarn.dist.archives", sparkRPath.getAbsolutePath()); - } else { - LOGGER.warn("sparkr.zip is not found, SparkR may not work."); - } - } - - private String getSparkMaster() { - String master = getJavaProperties().getProperty("master"); - if (master == null) { - master = getJavaProperties().getProperty("spark.master", "local[*]"); - } - return master; - } - - private String getDeployMode() { - String master = getSparkMaster(); - if (master.equals("yarn-client")) { - return "client"; - } else if (master.equals("yarn-cluster")) { - return "cluster"; - } else if (master.startsWith("local")) { - return "client"; - } else { - String deployMode = getJavaProperties().getProperty("spark.submit.deployMode"); - if (deployMode == null) { - throw new RuntimeException("master is set as yarn, but spark.submit.deployMode " + - "is not specified"); - } - if (!deployMode.equals("client") && !deployMode.equals("cluster")) { - throw new RuntimeException("Invalid value for spark.submit.deployMode: " + deployMode); - } - return deployMode; - } - } - - private boolean isYarnMode() { - return getSparkMaster().startsWith("yarn"); - } - - private String toShellFormat(String value) { - if (value.contains("\'") && value.contains("\"")) { - throw new RuntimeException("Spark property value could not contain both \" and '"); - } else if (value.contains("\'")) { - return "\"" + value + "\""; - } else { - return "\'" + value + "\'"; + synchronized RemoteInterpreterProcess createInterpreterProcess() throws IOException { + if (launcher == null) { + createLauncher(); } + InterpreterLaunchContext launchContext = new + InterpreterLaunchContext(getJavaProperties(), option, interpreterRunner, id, name); + RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext); + process.setRemoteInterpreterEventPoller( + new RemoteInterpreterEventPoller(remoteInterpreterProcessListener, appEventListener)); + return process; } private List getOrCreateSession(String user, String noteId) { @@ -881,18 +694,11 @@ public class InterpreterSetting { return null; } - private ManagedInterpreterGroup createInterpreterGroup(String groupId) - throws InterpreterException { + private ManagedInterpreterGroup createInterpreterGroup(String groupId) { AngularObjectRegistry angularObjectRegistry; ManagedInterpreterGroup interpreterGroup = new ManagedInterpreterGroup(groupId, this); - if (option.isRemote()) { - angularObjectRegistry = - new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup); - } else { - angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener); - // TODO(moon) : create distributed resource pool for local interpreters and set - } - + angularObjectRegistry = + new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup); interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); return interpreterGroup; } @@ -1010,7 +816,8 @@ public class InterpreterSetting { ); newProperties.put(key, property); } else { - throw new RuntimeException("Can not convert this type of property: " + value.getClass()); + throw new RuntimeException("Can not convert this type of property: " + + value.getClass()); } } return newProperties; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 73bababdf5..f34195d706 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -124,7 +124,7 @@ public class InterpreterSettingManager { remoteInterpreterProcessListener, ApplicationEventListener appEventListener) throws IOException { - this(zeppelinConfiguration, new InterpreterOption(true), + this(zeppelinConfiguration, new InterpreterOption(), angularObjectRegistryListener, remoteInterpreterProcessListener, appEventListener); @@ -192,8 +192,15 @@ public class InterpreterSettingManager { Map mergedProperties = new HashMap<>(InterpreterSetting.convertInterpreterProperties( interpreterSettingTemplate.getProperties())); - mergedProperties.putAll(InterpreterSetting.convertInterpreterProperties( - savedInterpreterSetting.getProperties())); + Map savedProperties = InterpreterSetting + .convertInterpreterProperties(savedInterpreterSetting.getProperties()); + for (Map.Entry entry : savedProperties.entrySet()) { + // only merge properties whose value is not empty + if (entry.getValue().getValue() != null && ! + StringUtils.isBlank(entry.getValue().toString())) { + mergedProperties.put(entry.getKey(), entry.getValue()); + } + } savedInterpreterSetting.setProperties(mergedProperties); // merge InterpreterInfo savedInterpreterSetting.setInterpreterInfos( @@ -737,11 +744,12 @@ public class InterpreterSettingManager { } /** - * Change interpreter property and restart + * Change interpreter properties and restart */ public void setPropertyAndRestart(String id, InterpreterOption option, Map properties, - List dependencies) throws IOException { + List dependencies) + throws InterpreterException, IOException { synchronized (interpreterSettings) { InterpreterSetting intpSetting = interpreterSettings.get(id); if (intpSetting != null) { @@ -754,7 +762,7 @@ public class InterpreterSettingManager { saveToFile(); } catch (Exception e) { loadFromFile(); - throw e; + throw new IOException(e); } } else { throw new InterpreterException("Interpreter setting id " + id + " not found"); @@ -763,7 +771,7 @@ public class InterpreterSettingManager { } // restart in note page - public void restart(String settingId, String noteId, String user) { + public void restart(String settingId, String noteId, String user) throws InterpreterException { InterpreterSetting intpSetting = interpreterSettings.get(settingId); Preconditions.checkNotNull(intpSetting); synchronized (interpreterSettings) { @@ -787,7 +795,7 @@ public class InterpreterSettingManager { } } - public void restart(String id) { + public void restart(String id) throws InterpreterException { restart(id, "", "anonymous"); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java index 1d7d916b6e..ff9cb1c47f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java @@ -25,6 +25,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.Collection; import java.util.List; @@ -52,7 +53,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return interpreterSetting; } - public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() { + public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() throws IOException { if (remoteInterpreterProcess == null) { LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId()); remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(); @@ -112,7 +113,11 @@ public class ManagedInterpreterGroup extends InterpreterGroup { LOGGER.info("Job " + job.getJobName() + " aborted "); } - interpreter.close(); + try { + interpreter.close(); + } catch (InterpreterException e) { + LOGGER.warn("Fail to close interpreter " + interpreter.getClassName(), e); + } //TODO(zjffdu) move the close of schedule to Interpreter if (null != scheduler) { SchedulerFactory.singleton().removeScheduler(scheduler.getName()); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java new file mode 100644 index 0000000000..f419967dee --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterRunner; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Interpreter Launcher which use shell script to launch the interpreter process. + * + */ +public class ShellScriptLauncher extends InterpreterLauncher { + + private static final Logger LOGGER = LoggerFactory.getLogger(ShellScriptLauncher.class); + + public ShellScriptLauncher(ZeppelinConfiguration zConf) { + super(zConf); + } + + @Override + public InterpreterClient launch(InterpreterLaunchContext context) { + LOGGER.info("Launching Interpreter: " + context.getInterpreterGroupName()); + this.properties = context.getProperties(); + InterpreterOption option = context.getOption(); + InterpreterRunner runner = context.getRunner(); + String groupName = context.getInterpreterGroupName(); + + int connectTimeout = + zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + if (option.isExistingProcess()) { + return new RemoteInterpreterRunningProcess( + connectTimeout, + option.getHost(), + option.getPort()); + } else { + // create new remote process + String localRepoPath = zConf.getInterpreterLocalRepoPath() + "/" + + context.getInterpreterGroupId(); + return new RemoteInterpreterManagedProcess( + runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(), + zConf.getCallbackPortRange(), + zConf.getInterpreterDir() + "/" + groupName, localRepoPath, + buildEnvFromProperties(), connectTimeout, groupName); + } + } + + protected Map buildEnvFromProperties() { + Map env = new HashMap<>(); + for (Object key : properties.keySet()) { + if (RemoteInterpreterUtils.isEnvString((String) key)) { + env.put((String) key, properties.getProperty((String) key)); + } + } + return env; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java new file mode 100644 index 0000000000..32a0530af1 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Spark specific launcher. + */ +public class SparkInterpreterLauncher extends ShellScriptLauncher { + + private static final Logger LOGGER = LoggerFactory.getLogger(SparkInterpreterLauncher.class); + + public SparkInterpreterLauncher(ZeppelinConfiguration zConf) { + super(zConf); + } + + @Override + protected Map buildEnvFromProperties() { + Map env = new HashMap(); + Properties sparkProperties = new Properties(); + String sparkMaster = getSparkMaster(properties); + for (String key : properties.stringPropertyNames()) { + if (RemoteInterpreterUtils.isEnvString(key)) { + env.put(key, properties.getProperty(key)); + } + if (isSparkConf(key, properties.getProperty(key))) { + sparkProperties.setProperty(key, toShellFormat(properties.getProperty(key))); + } + } + + setupPropertiesForPySpark(sparkProperties); + setupPropertiesForSparkR(sparkProperties); + if (isYarnMode() && getDeployMode().equals("cluster")) { + env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true"); + } + + StringBuilder sparkConfBuilder = new StringBuilder(); + if (sparkMaster != null) { + sparkConfBuilder.append(" --master " + sparkMaster); + } + if (isYarnMode() && getDeployMode().equals("cluster")) { + sparkConfBuilder.append(" --files " + zConf.getConfDir() + "/log4j_yarn_cluster.properties"); + } + for (String name : sparkProperties.stringPropertyNames()) { + sparkConfBuilder.append(" --conf " + name + "=" + sparkProperties.getProperty(name)); + } + + env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString()); + + // set these env in the order of + // 1. interpreter-setting + // 2. zeppelin-env.sh + // It is encouraged to set env in interpreter setting, but just for backward compatability, + // we also fallback to zeppelin-env.sh if it is not specified in interpreter setting. + for (String envName : new String[]{"SPARK_HOME", "SPARK_CONF_DIR", "HADOOP_CONF_DIR"}) { + String envValue = getEnv(envName); + if (envValue != null) { + env.put(envName, envValue); + } + } + LOGGER.debug("buildEnvFromProperties: " + env); + return env; + + } + + + /** + * get environmental variable in the following order + * + * 1. interpreter setting + * 2. zeppelin-env.sh + * + */ + private String getEnv(String envName) { + String env = properties.getProperty(envName); + if (env == null) { + env = System.getenv(envName); + } + return env; + } + + private boolean isSparkConf(String key, String value) { + return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value); + } + + private void setupPropertiesForPySpark(Properties sparkProperties) { + if (isYarnMode()) { + sparkProperties.setProperty("spark.yarn.isPython", "true"); + } + } + + private void mergeSparkProperty(Properties sparkProperties, String propertyName, + String propertyValue) { + if (sparkProperties.containsKey(propertyName)) { + String oldPropertyValue = sparkProperties.getProperty(propertyName); + sparkProperties.setProperty(propertyName, oldPropertyValue + "," + propertyValue); + } else { + sparkProperties.setProperty(propertyName, propertyValue); + } + } + + private void setupPropertiesForSparkR(Properties sparkProperties) { + String sparkHome = getEnv("SPARK_HOME"); + File sparkRBasePath = null; + if (sparkHome == null) { + if (!getSparkMaster(properties).startsWith("local")) { + throw new RuntimeException("SPARK_HOME is not specified in interpreter-setting" + + " for non-local mode, if you specify it in zeppelin-env.sh, please move that into " + + " interpreter setting"); + } + String zeppelinHome = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME); + sparkRBasePath = new File(zeppelinHome, + "interpreter" + File.separator + "spark" + File.separator + "R"); + } else { + sparkRBasePath = new File(sparkHome, "R" + File.separator + "lib"); + } + + File sparkRPath = new File(sparkRBasePath, "sparkr.zip"); + if (sparkRPath.exists() && sparkRPath.isFile()) { + mergeSparkProperty(sparkProperties, "spark.yarn.dist.archives", sparkRPath.getAbsolutePath()); + } else { + LOGGER.warn("sparkr.zip is not found, SparkR may not work."); + } + } + + /** + * Order to look for spark master + * 1. master in interpreter setting + * 2. spark.master interpreter setting + * 3. use local[*] + * @param properties + * @return + */ + private String getSparkMaster(Properties properties) { + String master = properties.getProperty("master"); + if (master == null) { + master = properties.getProperty("spark.master"); + if (master == null) { + master = "local[*]"; + } + } + return master; + } + + private String getDeployMode() { + String master = getSparkMaster(properties); + if (master.equals("yarn-client")) { + return "client"; + } else if (master.equals("yarn-cluster")) { + return "cluster"; + } else if (master.startsWith("local")) { + return "client"; + } else { + String deployMode = properties.getProperty("spark.submit.deployMode"); + if (deployMode == null) { + throw new RuntimeException("master is set as yarn, but spark.submit.deployMode " + + "is not specified"); + } + if (!deployMode.equals("client") && !deployMode.equals("cluster")) { + throw new RuntimeException("Invalid value for spark.submit.deployMode: " + deployMode); + } + return deployMode; + } + } + + private boolean isYarnMode() { + return getSparkMaster(properties).startsWith("yarn"); + } + + private String toShellFormat(String value) { + if (value.contains("\'") && value.contains("\"")) { + throw new RuntimeException("Spark property value could not contain both \" and '"); + } else if (value.contains("\'")) { + return "\"" + value + "\""; + } else { + return "\'" + value + "\'"; + } + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java index 064abd5370..7653824579 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java @@ -82,7 +82,7 @@ public class InterpreterContextRunnerPool { } } - throw new InterpreterException("Can not run paragraph " + paragraphId + " on " + noteId); + throw new RuntimeException("Can not run paragraph " + paragraphId + " on " + noteId); } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 54bf9e12db..b479799e24 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -28,6 +28,7 @@ import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.ManagedInterpreterGroup; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -42,6 +43,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -90,7 +92,7 @@ public class RemoteInterpreter extends Interpreter { return this.sessionId; } - public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() { + public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() throws IOException { if (this.interpreterProcess != null) { return this.interpreterProcess; } @@ -113,7 +115,7 @@ public class RemoteInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { synchronized (this) { if (!isOpened) { // create all the interpreters of the same session first, then Open the internal interpreter @@ -123,7 +125,11 @@ public class RemoteInterpreter extends Interpreter { // also see method Interpreter.getInterpreterInTheSameSessionByClassName for (Interpreter interpreter : getInterpreterGroup() .getOrCreateSession(userName, sessionId)) { - ((RemoteInterpreter) interpreter).internal_create(); + try { + ((RemoteInterpreter) interpreter).internal_create(); + } catch (IOException e) { + throw new InterpreterException(e); + } } interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction() { @@ -147,7 +153,7 @@ public class RemoteInterpreter extends Interpreter { } } - private void internal_create() { + private void internal_create() throws IOException { synchronized (this) { if (!isCreated) { RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); @@ -156,7 +162,7 @@ public class RemoteInterpreter extends Interpreter { public Void call(Client client) throws Exception { LOGGER.info("Create RemoteInterpreter {}", getClassName()); client.createInterpreter(getInterpreterGroup().getId(), sessionId, - className, (Map) property, userName); + className, (Map) getProperties(), userName); return null; } }); @@ -167,9 +173,14 @@ public class RemoteInterpreter extends Interpreter { @Override - public void close() { + public void close() throws InterpreterException { if (isOpened) { - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction() { @Override public Void call(Client client) throws Exception { @@ -184,13 +195,19 @@ public class RemoteInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(final String st, final InterpreterContext context) { + public InterpreterResult interpret(final String st, final InterpreterContext context) + throws InterpreterException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("st:\n{}", st); } final FormType form = getFormType(); - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess .getInterpreterContextRunnerPool(); List runners = context.getRunners(); @@ -238,12 +255,17 @@ public class RemoteInterpreter extends Interpreter { } @Override - public void cancel(final InterpreterContext context) { + public void cancel(final InterpreterContext context) throws InterpreterException { if (!isOpened) { LOGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className); return; } - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction() { @Override public Void call(Client client) throws Exception { @@ -254,7 +276,7 @@ public class RemoteInterpreter extends Interpreter { } @Override - public FormType getFormType() { + public FormType getFormType() throws InterpreterException { if (formType != null) { return formType; } @@ -265,7 +287,12 @@ public class RemoteInterpreter extends Interpreter { open(); } } - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } FormType type = interpreterProcess.callRemoteFunction( new RemoteInterpreterProcess.RemoteFunction() { @Override @@ -277,13 +304,19 @@ public class RemoteInterpreter extends Interpreter { return type; } + @Override - public int getProgress(final InterpreterContext context) { + public int getProgress(final InterpreterContext context) throws InterpreterException { if (!isOpened) { LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className); return 0; } - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } return interpreterProcess.callRemoteFunction( new RemoteInterpreterProcess.RemoteFunction() { @Override @@ -296,12 +329,18 @@ public class RemoteInterpreter extends Interpreter { @Override public List completion(final String buf, final int cursor, - final InterpreterContext interpreterContext) { + final InterpreterContext interpreterContext) + throws InterpreterException { if (!isOpened) { LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className); return new ArrayList<>(); } - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new InterpreterException(e); + } return interpreterProcess.callRemoteFunction( new RemoteInterpreterProcess.RemoteFunction>() { @Override @@ -317,7 +356,12 @@ public class RemoteInterpreter extends Interpreter { LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className); return Job.Status.UNKNOWN.name(); } - RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); + RemoteInterpreterProcess interpreterProcess = null; + try { + interpreterProcess = getOrCreateInterpreterProcess(); + } catch (IOException e) { + throw new RuntimeException(e); + } return interpreterProcess.callRemoteFunction( new RemoteInterpreterProcess.RemoteFunction() { @Override @@ -331,7 +375,7 @@ public class RemoteInterpreter extends Interpreter { @Override public Scheduler getScheduler() { int maxConcurrency = Integer.parseInt( - property.getProperty("zeppelin.interpreter.max.poolsize", + getProperty("zeppelin.interpreter.max.poolsize", ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + "")); Scheduler s = new RemoteScheduler( diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index d21a9623d0..6e26e58f8d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.interpreter.remote; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -73,11 +74,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess String localRepoDir, Map env, int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener, String interpreterGroupName) { - super(new RemoteInterpreterEventPoller(listener, appListener), - connectTimeout); + super(connectTimeout); this.interpreterRunner = intpRunner; this.portRange = portRange; this.env = env; @@ -86,23 +84,6 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess this.interpreterGroupName = interpreterGroupName; } - RemoteInterpreterManagedProcess(String intpRunner, - String intpDir, - String localRepoDir, - Map env, - RemoteInterpreterEventPoller remoteInterpreterEventPoller, - int connectTimeout, - String interpreterGroupName) { - super(remoteInterpreterEventPoller, - connectTimeout); - this.interpreterRunner = intpRunner; - this.portRange = ":"; - this.env = env; - this.interpreterDir = intpDir; - this.localRepoDir = localRepoDir; - this.interpreterGroupName = interpreterGroupName; - } - @Override public String getHost() { return "localhost"; @@ -124,7 +105,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess callbackHost = RemoteInterpreterUtils.findAvailableHostAddress(); callbackPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); } catch (IOException e1) { - throw new InterpreterException(e1); + throw new RuntimeException(e1); } logger.info("Thrift server for callback will start. Port: {}", callbackPort); @@ -206,7 +187,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess executor.execute(cmdLine, procEnv, this); } catch (IOException e) { running.set(false); - throw new InterpreterException(e); + throw new RuntimeException(e); } try { @@ -217,7 +198,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess } if (!running.get()) { callbackServer.stop(); - throw new InterpreterException("Cannot run interpreter"); + throw new RuntimeException(new String(cmdOut.toByteArray())); } } catch (InterruptedException e) { logger.error("Remote interpreter is not accessible"); @@ -227,7 +208,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess public void stop() { // shutdown EventPoller first. - this.remoteInterpreterEventPoller.shutdown(); + this.getRemoteInterpreterEventPoller().shutdown(); if (callbackServer.isServing()) { callbackServer.stop(); } @@ -266,6 +247,31 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess running.set(false); } + @VisibleForTesting + public Map getEnv() { + return env; + } + + @VisibleForTesting + public String getLocalRepoDir() { + return localRepoDir; + } + + @VisibleForTesting + public String getInterpreterDir() { + return interpreterDir; + } + + @VisibleForTesting + public String getInterpreterGroupName() { + return interpreterGroupName; + } + + @VisibleForTesting + public String getInterpreterRunner() { + return interpreterRunner; + } + public boolean isRunning() { return running.get(); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index e45f15b109..88cc4894be 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -21,6 +21,7 @@ import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.thrift.TException; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.launcher.InterpreterClient; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,27 +29,17 @@ import org.slf4j.LoggerFactory; /** * Abstract class for interpreter process */ -public abstract class RemoteInterpreterProcess { +public abstract class RemoteInterpreterProcess implements InterpreterClient { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class); private GenericObjectPool clientPool; - protected final RemoteInterpreterEventPoller remoteInterpreterEventPoller; + private RemoteInterpreterEventPoller remoteInterpreterEventPoller; private final InterpreterContextRunnerPool interpreterContextRunnerPool; private int connectTimeout; public RemoteInterpreterProcess( - int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener) { - this(new RemoteInterpreterEventPoller(listener, appListener), - connectTimeout); - this.remoteInterpreterEventPoller.setInterpreterProcess(this); - } - - RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller, - int connectTimeout) { + int connectTimeout) { this.interpreterContextRunnerPool = new InterpreterContextRunnerPool(); - this.remoteInterpreterEventPoller = remoteInterpreterEventPoller; this.connectTimeout = connectTimeout; } @@ -56,6 +47,10 @@ public abstract class RemoteInterpreterProcess { return remoteInterpreterEventPoller; } + public void setRemoteInterpreterEventPoller(RemoteInterpreterEventPoller eventPoller) { + this.remoteInterpreterEventPoller = eventPoller; + } + public abstract String getHost(); public abstract int getPort(); public abstract void start(String userName, Boolean isUserImpersonate); @@ -147,9 +142,9 @@ public abstract class RemoteInterpreterProcess { } } catch (TException e) { broken = true; - throw new InterpreterException(e); + throw new RuntimeException(e); } catch (Exception e1) { - throw new InterpreterException(e1); + throw new RuntimeException(e1); } finally { if (client != null) { releaseClient(client, broken); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java index bb176bea59..d8715a0d49 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java @@ -30,12 +30,10 @@ public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess { public RemoteInterpreterRunningProcess( int connectTimeout, - RemoteInterpreterProcessListener listener, - ApplicationEventListener appListener, String host, int port ) { - super(connectTimeout, listener, appListener); + super(connectTimeout); this.host = host; this.port = port; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 03c50468c5..b5dda67342 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -627,7 +627,7 @@ public class Note implements ParagraphJobListener, JsonSerializable { if (intp == null) { String intpExceptionMsg = p.getJobName() + "'s Interpreter " + requiredReplName + " not found"; - InterpreterException intpException = new InterpreterException(intpExceptionMsg); + RuntimeException intpException = new RuntimeException(intpExceptionMsg); InterpreterResult intpResult = new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage()); p.setReturn(intpResult, intpException); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 4652fcd804..77fd04ca7b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -892,7 +892,11 @@ public class Notebook implements NoteEventListener { if (releaseResource) { for (InterpreterSetting setting : notebook.getInterpreterSettingManager() .getInterpreterSettings(note.getId())) { - notebook.getInterpreterSettingManager().restart(setting.getId()); + try { + notebook.getInterpreterSettingManager().restart(setting.getId()); + } catch (InterpreterException e) { + logger.error("Fail to restart interpreter: " + setting.getId(), e); + } } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 161dc307a5..701943a821 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -26,7 +26,7 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Random; +import java.security.SecureRandom; import java.util.Set; import org.apache.commons.lang.StringUtils; @@ -139,8 +139,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } private static String generateId() { - return "paragraph_" + System.currentTimeMillis() + "_" + new Random(System.currentTimeMillis()) - .nextInt(); + return "paragraph_" + System.currentTimeMillis() + "_" + new SecureRandom().nextInt(); } public Map getUserParagraphMap() { @@ -313,15 +312,14 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { String replName = getRequiredReplName(trimmedBuffer); String body = getScriptBody(trimmedBuffer); - Interpreter repl = getRepl(replName); - if (repl == null) { - return null; - } - InterpreterContext interpreterContext = getInterpreterContextWithoutRunner(null); - List completion = repl.completion(body, cursor, interpreterContext); - return completion; + try { + Interpreter repl = getRepl(replName); + return repl.completion(body, cursor, interpreterContext); + } catch (InterpreterException e) { + throw new RuntimeException("Fail to get completion", e); + } } public int calculateCursorPosition(String buffer, String trimmedBuffer, int cursor) { @@ -363,11 +361,15 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { @Override public int progress() { String replName = getRequiredReplName(); - Interpreter repl = getRepl(replName); - if (repl != null) { + + try { + Interpreter repl = getRepl(replName); + if (repl == null) { + return 0; + } return repl.getProgress(getInterpreterContext(null)); - } else { - return 0; + } catch (InterpreterException e) { + throw new RuntimeException("Fail to get progress", e); } } @@ -495,10 +497,8 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { protected boolean jobAbort() { Interpreter repl = getRepl(getRequiredReplName()); if (repl == null) { - // when interpreters are already destroyed return true; } - Scheduler scheduler = repl.getScheduler(); if (scheduler == null) { return true; @@ -508,7 +508,11 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { if (job != null) { job.setStatus(Status.ABORT); } else { - repl.cancel(getInterpreterContextWithoutRunner(null)); + try { + repl.cancel(getInterpreterContextWithoutRunner(null)); + } catch (InterpreterException e) { + throw new RuntimeException(e); + } } return true; } @@ -739,12 +743,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } private boolean isValidInterpreter(String replName) { - try { - return factory.getInterpreter(user, note.getId(), replName) != null; - } catch (InterpreterException e) { - // ignore this exception, it would be recaught when running paragraph. - return false; - } + return factory.getInterpreter(user, note.getId(), replName) != null; } public void updateRuntimeInfos(String label, String tooltip, Map infos, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java index 150ac26ce9..ba858e6925 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; @@ -50,8 +49,10 @@ public class FileSystemNotebookRepo implements NotebookRepo { this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); if (isSecurityEnabled) { - String keytab = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_HDFS_KEYTAB); - String principal = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_HDFS_PRINCIPAL); + String keytab = zConf.getString( + ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_KEYTAB); + String principal = zConf.getString( + ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_KERBEROS_PRINCIPAL); if (StringUtils.isBlank(keytab) || StringUtils.isBlank(principal)) { throw new IOException("keytab and principal can not be empty, keytab: " + keytab + ", principal: " + principal); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java index fd5142bd48..38d8b5005a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java @@ -47,7 +47,6 @@ public class Authentication implements Runnable { // Cipher is an AES in CBC mode private static final String CIPHER_ALGORITHM = "AES"; private static final String CIPHER_MODE = "AES/CBC/PKCS5PADDING"; - private static final String KEY = "AbtEr99DxsWWbJkP"; private static final int ivSize = 16; private static final String ZEPPELIN_CONF_ANONYMOUS_ALLOWED = "zeppelin.anonymous.allowed"; @@ -198,7 +197,7 @@ public class Authentication implements Runnable { private Key generateKey() { try { KeyGenerator kgen = KeyGenerator.getInstance(CIPHER_ALGORITHM); - kgen.init(128, new SecureRandom(toBytes(KEY))); + kgen.init(128, new SecureRandom()); SecretKey secretKey = kgen.generateKey(); byte[] enCodeFormat = secretKey.getEncoded(); return new SecretKeySpec(enCodeFormat, CIPHER_ALGORITHM); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java index 98aaac8a75..7b0d804de9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java @@ -18,9 +18,9 @@ package org.apache.zeppelin.notebook.utility; import java.math.BigInteger; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; -import java.util.Random; /** * Generate Tiny ID. @@ -71,6 +71,6 @@ public class IdHashes { } public static String generateId() { - return encode(System.currentTimeMillis() + new Random().nextInt()); + return encode(System.currentTimeMillis() + new SecureRandom().nextInt()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index bf494908a0..f23d43380d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -19,6 +19,7 @@ package org.apache.zeppelin.helium; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.AbstractInterpreterTest; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.ApplicationState; @@ -77,7 +78,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implem this, search, notebookAuthorization, - new Credentials(false, null)); + new Credentials(false, null, null)); heliumAppFactory.setNotebook(notebook); @@ -241,7 +242,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implem @Test - public void testUnloadOnInterpreterRestart() throws IOException { + public void testUnloadOnInterpreterRestart() throws IOException, InterpreterException { // given HeliumPackage pkg1 = new HeliumPackage(HeliumType.APPLICATION, "name1", diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java index e7a04f3b5f..cf1713ff1b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java @@ -41,7 +41,7 @@ public class EchoInterpreter extends Interpreter { @Override public InterpreterResult interpret(String st, InterpreterContext context) { - if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) { + if (Boolean.parseBoolean(getProperty("zeppelin.interpreter.echo.fail", "false"))) { return new InterpreterResult(InterpreterResult.Code.ERROR); } else { return new InterpreterResult(InterpreterResult.Code.SUCCESS, st); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index f3137d9c89..abbaa3520e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -52,10 +52,10 @@ public class InterpreterFactoryTest extends AbstractInterpreterTest { assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName()); } - @Test(expected = InterpreterException.class) + @Test public void testUnknownRepl1() throws IOException { interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds()); - interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl"); + assertNull(interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl")); } @Test diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java index 0bcdb6fd5a..605476fe32 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java @@ -96,7 +96,7 @@ public class InterpreterSettingManagerTest extends AbstractInterpreterTest { } @Test - public void testCreateUpdateRemoveSetting() throws IOException { + public void testCreateUpdateRemoveSetting() throws IOException, InterpreterException { // create new interpreter setting InterpreterOption option = new InterpreterOption(); option.setPerNote("scoped"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java index 3c061a99a4..e3e47d3b5c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java @@ -32,7 +32,6 @@ public class InterpreterSettingTest { public void testCreateInterpreters() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SHARED); - interpreterOption.setRemote(false); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -64,7 +63,6 @@ public class InterpreterSettingTest { public void testSharedMode() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SHARED); - interpreterOption.setRemote(false); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -101,7 +99,6 @@ public class InterpreterSettingTest { public void testPerUserScopedMode() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SCOPED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -138,7 +135,6 @@ public class InterpreterSettingTest { public void testPerNoteScopedMode() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerNote(InterpreterOption.SCOPED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -175,7 +171,6 @@ public class InterpreterSettingTest { public void testPerUserIsolatedMode() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.ISOLATED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -212,7 +207,6 @@ public class InterpreterSettingTest { public void testPerNoteIsolatedMode() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerNote(InterpreterOption.ISOLATED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -249,7 +243,6 @@ public class InterpreterSettingTest { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.ISOLATED); interpreterOption.setPerNote(InterpreterOption.SCOPED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -301,7 +294,6 @@ public class InterpreterSettingTest { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.ISOLATED); interpreterOption.setPerNote(InterpreterOption.ISOLATED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); @@ -358,7 +350,6 @@ public class InterpreterSettingTest { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SCOPED); interpreterOption.setPerNote(InterpreterOption.SCOPED); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java index a5d7bd4359..74bd201085 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java @@ -41,7 +41,6 @@ public class ManagedInterpreterGroupTest { public void setUp() throws IOException, RepositoryException { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SCOPED); - interpreterOption.setRemote(false); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); List interpreterInfos = new ArrayList<>(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java index 9deafcfa00..7a904c6bfb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java @@ -46,7 +46,7 @@ public class SleepInterpreter extends Interpreter { @Override public Scheduler getScheduler() { - if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) { + if (Boolean.parseBoolean(getProperty("zeppelin.SleepInterpreter.parallel", "false"))) { return SchedulerFactory.singleton().createOrGetParallelScheduler( "Parallel-" + SleepInterpreter.class.getName(), 10); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java index 24a9aeed22..930a26d9b6 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java @@ -44,7 +44,7 @@ public class SparkInterpreterModeTest { } } - private void testInterpreterBasics() throws IOException { + private void testInterpreterBasics() throws IOException, InterpreterException { // test SparkInterpreter interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds()); Interpreter sparkInterpreter = interpreterFactory.getInterpreter("user1", "note1", "spark.spark"); @@ -75,7 +75,7 @@ public class SparkInterpreterModeTest { } @Test - public void testLocalMode() throws IOException, YarnException { + public void testLocalMode() throws IOException, YarnException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "local[*]"); sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); @@ -94,7 +94,7 @@ public class SparkInterpreterModeTest { } @Test - public void testYarnClientMode() throws IOException, YarnException, InterruptedException { + public void testYarnClientMode() throws IOException, YarnException, InterruptedException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-client"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); @@ -116,7 +116,7 @@ public class SparkInterpreterModeTest { } @Test - public void testYarnClusterMode() throws IOException, YarnException, InterruptedException { + public void testYarnClusterMode() throws IOException, YarnException, InterruptedException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-cluster"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java new file mode 100644 index 0000000000..9708ee7d56 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ShellScriptLauncherTest { + + @Test + public void testLauncher() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + ShellScriptLauncher launcher = new ShellScriptLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("ENV_1", "VALUE_1"); + properties.setProperty("property_1", "value_1"); + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "groupName"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("groupName", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(1, interpreterProcess.getEnv().size()); + assertEquals("VALUE_1", interpreterProcess.getEnv().get("ENV_1")); + } + +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java new file mode 100644 index 0000000000..45e009f719 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.launcher; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; +import org.junit.Test; + +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SparkInterpreterLauncherTest { + + @Test + public void testLocalMode() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("SPARK_HOME", "/user/spark"); + properties.setProperty("property_1", "value_1"); + properties.setProperty("master", "local[*]"); + properties.setProperty("spark.files", "file_1"); + properties.setProperty("spark.jars", "jar_1"); + + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(2, interpreterProcess.getEnv().size()); + assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); + assertEquals(" --master local[*] --conf spark.files='file_1' --conf spark.jars='jar_1'", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + } + + @Test + public void testYarnClientMode_1() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("SPARK_HOME", "/user/spark"); + properties.setProperty("property_1", "value_1"); + properties.setProperty("master", "yarn-client"); + properties.setProperty("spark.files", "file_1"); + properties.setProperty("spark.jars", "jar_1"); + + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(2, interpreterProcess.getEnv().size()); + assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); + assertEquals(" --master yarn-client --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + } + + @Test + public void testYarnClientMode_2() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("SPARK_HOME", "/user/spark"); + properties.setProperty("property_1", "value_1"); + properties.setProperty("master", "yarn"); + properties.setProperty("spark.submit.deployMode", "client"); + properties.setProperty("spark.files", "file_1"); + properties.setProperty("spark.jars", "jar_1"); + + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(2, interpreterProcess.getEnv().size()); + assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); + assertEquals(" --master yarn --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='client' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + } + + @Test + public void testYarnClusterMode_1() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("SPARK_HOME", "/user/spark"); + properties.setProperty("property_1", "value_1"); + properties.setProperty("master", "yarn-cluster"); + properties.setProperty("spark.files", "file_1"); + properties.setProperty("spark.jars", "jar_1"); + + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(3, interpreterProcess.getEnv().size()); + assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); + assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER")); + assertEquals(" --master yarn-cluster --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + } + + @Test + public void testYarnClusterMode_2() { + ZeppelinConfiguration zConf = new ZeppelinConfiguration(); + SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf); + Properties properties = new Properties(); + properties.setProperty("SPARK_HOME", "/user/spark"); + properties.setProperty("property_1", "value_1"); + properties.setProperty("master", "yarn"); + properties.setProperty("spark.submit.deployMode", "cluster"); + properties.setProperty("spark.files", "file_1"); + properties.setProperty("spark.jars", "jar_1"); + + InterpreterOption option = new InterpreterOption(); + InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark"); + InterpreterClient client = launcher.launch(context); + assertTrue( client instanceof RemoteInterpreterManagedProcess); + RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; + assertEquals("spark", interpreterProcess.getInterpreterGroupName()); + assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir()); + assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); + assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); + assertEquals(3, interpreterProcess.getEnv().size()); + assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); + assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER")); + assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 61e4ef06c0..54814c45ec 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -60,7 +60,6 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { onRemove = new AtomicInteger(0); InterpreterOption interpreterOption = new InterpreterOption(); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap()); List interpreterInfos = new ArrayList<>(); interpreterInfos.add(interpreterInfo1); @@ -101,7 +100,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { } @Test - public void testAngularObjectInterpreterSideCRUD() throws InterruptedException { + public void testAngularObjectInterpreterSideCRUD() throws InterruptedException, InterpreterException { InterpreterResult ret = intp.interpret("get", context); Thread.sleep(500); // waitFor eventpoller pool event String[] result = ret.message().get(0).getData().split(" "); @@ -134,7 +133,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { } @Test - public void testAngularObjectRemovalOnZeppelinServerSide() throws InterruptedException { + public void testAngularObjectRemovalOnZeppelinServerSide() throws InterruptedException, InterpreterException { // test if angularobject removal from server side propagate to interpreter process's registry. // will happen when notebook is removed. @@ -159,7 +158,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener { } @Test - public void testAngularObjectAddOnZeppelinServerSide() throws InterruptedException { + public void testAngularObjectAddOnZeppelinServerSide() throws InterruptedException, InterpreterException { // test if angularobject add from server side propagate to interpreter process's registry. // will happen when zeppelin server loads notebook and restore the object into registry diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 1687060d6d..f52803d2dc 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -46,7 +46,6 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce public void setUp() throws Exception { InterpreterOption interpreterOption = new InterpreterOption(); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterOutputStream.class.getName(), "mock", true, new HashMap()); List interpreterInfos = new ArrayList<>(); interpreterInfos.add(interpreterInfo1); @@ -83,7 +82,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce } @Test - public void testInterpreterResultOnly() { + public void testInterpreterResultOnly() throws InterpreterException { RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); @@ -99,7 +98,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce } @Test - public void testInterpreterOutputStreamOnly() { + public void testInterpreterOutputStreamOnly() throws InterpreterException { RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); @@ -111,7 +110,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce } @Test - public void testInterpreterResultOutputStreamMixed() { + public void testInterpreterResultOutputStreamMixed() throws InterpreterException { RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); @@ -120,7 +119,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce } @Test - public void testOutputType() { + public void testOutputType() throws InterpreterException { RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 0a5e484979..74bde89dfa 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -52,7 +52,6 @@ public class RemoteInterpreterTest { public void setUp() throws Exception { InterpreterOption interpreterOption = new InterpreterOption(); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); InterpreterInfo interpreterInfo3 = new InterpreterInfo(SleepInterpreter.class.getName(), "sleep", false, new HashMap()); @@ -82,7 +81,7 @@ public class RemoteInterpreterTest { } @Test - public void testSharedMode() { + public void testSharedMode() throws InterpreterException, IOException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1"); @@ -125,7 +124,7 @@ public class RemoteInterpreterTest { } @Test - public void testScopedMode() { + public void testScopedMode() throws InterpreterException, IOException { interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED); Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1"); @@ -171,7 +170,7 @@ public class RemoteInterpreterTest { } @Test - public void testIsolatedMode() { + public void testIsolatedMode() throws InterpreterException, IOException { interpreterSetting.getOption().setPerUser(InterpreterOption.ISOLATED); Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1"); @@ -218,7 +217,7 @@ public class RemoteInterpreterTest { } @Test - public void testExecuteIncorrectPrecode() throws TTransportException, IOException { + public void testExecuteIncorrectPrecode() throws TTransportException, IOException, InterpreterException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); interpreterSetting.setProperty("zeppelin.SleepInterpreter.precode", "fail test"); Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep"); @@ -229,7 +228,7 @@ public class RemoteInterpreterTest { } @Test - public void testExecuteCorrectPrecode() throws TTransportException, IOException { + public void testExecuteCorrectPrecode() throws TTransportException, IOException, InterpreterException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); interpreterSetting.setProperty("zeppelin.SleepInterpreter.precode", "1"); Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep"); @@ -240,7 +239,7 @@ public class RemoteInterpreterTest { } @Test - public void testRemoteInterperterErrorStatus() throws TTransportException, IOException { + public void testRemoteInterperterErrorStatus() throws TTransportException, IOException, InterpreterException { interpreterSetting.setProperty("zeppelin.interpreter.echo.fail", "true"); interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); @@ -255,7 +254,7 @@ public class RemoteInterpreterTest { } @Test - public void testFIFOScheduler() throws InterruptedException { + public void testFIFOScheduler() throws InterruptedException, InterpreterException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); // by default SleepInterpreter would use FIFOScheduler @@ -269,13 +268,23 @@ public class RemoteInterpreterTest { Thread thread1 = new Thread() { @Override public void run() { - assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + try { + assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + } catch (InterpreterException e) { + e.printStackTrace(); + fail(); + } } }; Thread thread2 = new Thread() { @Override public void run() { - assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + try { + assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + } catch (InterpreterException e) { + e.printStackTrace(); + fail(); + } } }; long start = System.currentTimeMillis(); @@ -288,7 +297,7 @@ public class RemoteInterpreterTest { } @Test - public void testParallelScheduler() throws InterruptedException { + public void testParallelScheduler() throws InterruptedException, InterpreterException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); interpreterSetting.setProperty("zeppelin.SleepInterpreter.parallel", "true"); @@ -303,13 +312,23 @@ public class RemoteInterpreterTest { Thread thread1 = new Thread() { @Override public void run() { - assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + try { + assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + } catch (InterpreterException e) { + e.printStackTrace(); + fail(); + } } }; Thread thread2 = new Thread() { @Override public void run() { - assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + try { + assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code()); + } catch (InterpreterException e) { + e.printStackTrace(); + fail(); + } } }; long start = System.currentTimeMillis(); @@ -377,7 +396,7 @@ public class RemoteInterpreterTest { } @Test - public void testEnvironmentAndProperty() { + public void testEnvironmentAndProperty() throws InterpreterException { interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED); interpreterSetting.setProperty("ENV_1", "VALUE_1"); interpreterSetting.setProperty("property_1", "value_1"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java index 5a3e57c06f..dbd2df77f5 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java @@ -51,8 +51,9 @@ public class MockInterpreterA extends Interpreter { } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - if (property.containsKey("progress")) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { + if (getProperties().containsKey("progress")) { context.setProgress(Integer.parseInt(getProperty("progress"))); } try { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java index 1890cbc94e..7a5321ae3a 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java @@ -52,7 +52,8 @@ public class MockInterpreterOutputStream extends Interpreter { } @Override - public InterpreterResult interpret(String st, InterpreterContext context) { + public InterpreterResult interpret(String st, InterpreterContext context) + throws InterpreterException { String[] ret = st.split(":"); try { if (ret[1] != null) { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index f044fbda5d..a1621e86ee 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -23,8 +23,6 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.AbstractInterpreterTest; -import org.apache.zeppelin.interpreter.ClassloaderInterpreter; -import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -32,9 +30,6 @@ import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.LazyOpenInterpreter; -import org.apache.zeppelin.interpreter.mock.MockInterpreter1; -import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; @@ -98,7 +93,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); - credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); + credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null); notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search, notebookAuthorization, credentials); @@ -667,7 +662,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener @Test public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedException, - IOException { + IOException, InterpreterException { // create a note and a paragraph Note note = notebook.createNote(anonymous); interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); @@ -795,7 +790,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener @Test public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, - IOException { + IOException, InterpreterException { Note note = notebook.createNote(anonymous); interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds()); @@ -829,7 +824,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener } @Test - public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { + public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException, InterpreterException { // create a notes Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -861,7 +856,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener } @Test - public void testPerSessionInterpreter() throws IOException { + public void testPerSessionInterpreter() throws IOException, InterpreterException { // create two notes Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -905,7 +900,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener @Test - public void testPerNoteSessionInterpreter() throws IOException { + public void testPerNoteSessionInterpreter() throws IOException, InterpreterException { // create two notes Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -964,7 +959,7 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener } @Test - public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IOException { + public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IOException, InterpreterException { // create a notes Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 8a0b5a13b7..22366545ee 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -103,7 +103,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); notebookAuthorization = NotebookAuthorization.init(conf); - credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); + credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null); notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, credentials); anonymous = new AuthenticationInfo("anonymous"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index 469d5b593c..c7ccd467a3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -79,7 +79,7 @@ public class DistributedResourcePoolTest extends AbstractInterpreterTest { } @Test - public void testRemoteDistributedResourcePool() { + public void testRemoteDistributedResourcePool() throws InterpreterException { Gson gson = new Gson(); InterpreterResult ret; intp1.interpret("put key1 value1", context); @@ -162,7 +162,7 @@ public class DistributedResourcePoolTest extends AbstractInterpreterTest { } @Test - public void testResourcePoolUtils() { + public void testResourcePoolUtils() throws InterpreterException { Gson gson = new Gson(); InterpreterResult ret; @@ -201,7 +201,7 @@ public class DistributedResourcePoolTest extends AbstractInterpreterTest { } @Test - public void testResourceInvokeMethod() { + public void testResourceInvokeMethod() throws InterpreterException { Gson gson = new Gson(); InterpreterResult ret; intp1.interpret("put key1 hey", context); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index a758591727..e43a39d52d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -20,6 +20,7 @@ package org.apache.zeppelin.scheduler; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterInfo; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -63,7 +64,6 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { schedulerSvc = new SchedulerFactory(); InterpreterOption interpreterOption = new InterpreterOption(); - interpreterOption.setRemote(true); InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap()); List interpreterInfos = new ArrayList<>(); interpreterInfos.add(interpreterInfo1); @@ -210,7 +210,11 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { @Override protected boolean jobAbort() { if (isRunning()) { - intpA.cancel(context); + try { + intpA.cancel(context); + } catch (InterpreterException e) { + e.printStackTrace(); + } } return true; } @@ -260,7 +264,11 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { @Override protected boolean jobAbort() { if (isRunning()) { - intpA.cancel(context); + try { + intpA.cancel(context); + } catch (InterpreterException e) { + e.printStackTrace(); + } } return true; }