Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-277

Browse source
This commit is contained in:
Malay Majithia 2017-10-16 03:08:31 +05:30
commit 18fc814d0a
No known key found for this signature in database
GPG key ID: 52850166EEE05697
204 changed files with 4950 additions and 2125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/PULL_REQUEST_TEMPLATE vendored
View file

@ -14,7 +14,9 @@ First time? Check out the contributing guide - https://zeppelin.apache.org/contr
* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533]
### How should this be tested?
Outline the steps to test the PR here.
* First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
### Screenshots (if appropriate)

4
.travis.yml
View file

@ -66,7 +66,8 @@ matrix:
# Several tests were excluded from this configuration due to the following issues:
# HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
# After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
- jdk: "oraclejdk8"
- sudo: required
jdk: "oraclejdk8"
dist: precise
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
@ -143,6 +144,7 @@ before_script:
- if [[ -n $LIVY_VER ]]; then ./testing/downloadLivy.sh $LIVY_VER; fi
- if [[ -n $LIVY_VER ]]; then export LIVY_HOME=`pwd`/livy-$LIVY_VER-bin; fi
- if [[ -n $LIVY_VER ]]; then export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER; fi
- export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
- echo "export ZEPPELIN_HELIUM_REGISTRY=helium" >> conf/zeppelin-env.sh
- tail conf/zeppelin-env.sh

10
README.md
View file

@ -1,7 +1,7 @@
# Apache Zeppelin
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
**Documentation:** [User Guide](https://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](https://zeppelin.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://travis-ci.org/apache/zeppelin.svg?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
@ -15,15 +15,15 @@ Core feature:
* Built-in Apache Spark support
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org)
To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](https://zeppelin.apache.org)
## Getting Started
### Install binary package
Please go to [install](http://zeppelin.apache.org/docs/snapshot/install/install.html) to install Apache Zeppelin from binary package.
Please go to [install](https://zeppelin.apache.org/docs/latest/install/install.html) to install Apache Zeppelin from binary package.
### Build from source
Please check [Build from source](http://zeppelin.apache.org/docs/snapshot/install/build.html) to build Zeppelin from source.
Please check [Build from source](https://zeppelin.apache.org/docs/latest/install/build.html) to build Zeppelin from source.

53
alluxio/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -34,6 +34,7 @@
<properties>
<alluxio.version>1.0.0</alluxio.version>
<interpreter.name>alluxio</interpreter.name>
</properties>
<dependencies>
@ -129,54 +130,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/alluxio</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

56
angular/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -32,6 +32,10 @@
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Angular interpreter</name>
<properties>
<interpreter.name>angular</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@ -61,54 +65,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/angular</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

64
beam/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -41,6 +41,7 @@
<netty.version>4.1.1.Final</netty.version>
<servlet.api.version>3.1.0</servlet.api.version>
<commons.exec.version>1.3</commons.exec.version>
<interpreter.name>beam</interpreter.name>
</properties>
<dependencies>
@ -240,69 +241,18 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/beam</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

85
bigquery/pom.xml
View file

@ -21,9 +21,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -41,6 +42,7 @@
<!-- library versions -->
<bigquery.api.version>v2-rev265-1.21.0</bigquery.api.version>
<gson.version>2.6</gson.version>
<interpreter.name>bigquery</interpreter.name>
</properties>
<dependencies>
@ -99,12 +101,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
@ -116,63 +118,22 @@
</excludes>
</configuration>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/bqsql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>
org.apache.zeppelin.bigquery.BigQueryInterpreter
</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>
org.apache.zeppelin.bigquery.BigQueryInterpreter
</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
</project>

6
bin/common.sh
View file

@ -122,7 +122,11 @@ JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS
JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
else
JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties"
fi
export JAVA_INTP_OPTS

13
bin/interpreter.sh
View file

@ -143,7 +143,13 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
fi
unset PYSPARKPATH
export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
else
# autodetect HADOOP_CONF_HOME by heuristic
if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then
if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then
@ -152,13 +158,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export HADOOP_CONF_DIR="/etc/hadoop/conf"
fi
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
fi
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
if [[ -n "${HBASE_CONF_DIR}" ]]; then
ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}"

54
cassandra/pom.xml
View file

@ -20,10 +20,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -49,6 +49,7 @@
<plugin.scala.version>2.15.2</plugin.scala.version>
<plugin.scalatest.version>1.0</plugin.scalatest.version>
<plugin.scalate.version>1.7.1</plugin.scalate.version>
<interpreter.name>cassandra</interpreter.name>
</properties>
<dependencies>
@ -241,55 +242,14 @@
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

23
conf/log4j_yarn_cluster.properties Normal file
View file

@ -0,0 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger = INFO, stdout
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n

8
conf/shiro.ini.template
View file

@ -62,6 +62,14 @@ sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
#securityManager.cacheManager = $cacheManager
### Enables 'HttpOnly' flag in Zeppelin cookies
cookie = org.apache.shiro.web.servlet.SimpleCookie
cookie.name = JSESSIONID
cookie.httpOnly = true
### Uncomment the below line only when Zeppelin is running over HTTPS
#cookie.secure = true
sessionManager.sessionIdCookie = $cookie
securityManager.sessionManager = $sessionManager
# 86,400,000 milliseconds = 24 hour
securityManager.sessionManager.globalSessionTimeout = 86400000

20
conf/zeppelin-site.xml.template
View file

@ -138,6 +138,16 @@
</property>
-->
<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
<!--
<property>
<name>zeppelin.notebook.s3.signerOverride</name>
<value>S3SignerType</value>
<description>optional override to control which signature algorithm should be used to sign AWS requests</description>
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>
@ -173,21 +183,21 @@
</property>
-->
<!-- Notebook storage layer using hdfs file system
<!-- Notebook storage layer using hadoop compatible file system
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
<description>hdfs notebook persistence layer implementation</description>
<value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
<description>Hadoop compatible file system notebook persistence layer implementation, such as local file system, hdfs, azure wasb, s3 and etc.</description>
</property>
<property>
<name>zeppelin.hdfs.keytab</name>
<name>zeppelin.server.kerberos.keytab</name>
<value></value>
<description>keytab for accessing kerberized hdfs</description>
</property>
<property>
<name>zeppelin.hdfs.principal</name>
<name>zeppelin.server.kerberos.principal</name>
<value></value>
<description>principal for accessing kerberized hdfs</description>
</property>

1
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -137,6 +137,7 @@
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/neo4j.html">Neo4j</a></li>
<li><a href="{{BASE_PATH}}/interpreter/pig.html">Pig</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, HAWQ</a></li>
<li><a href="{{BASE_PATH}}/interpreter/r.html">R</a></li>

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-config.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 21 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-graph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 29 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.5 MiB

44
docs/development/writing_zeppelin_interpreter.md
View file

@ -40,7 +40,49 @@ In 'Separate Interpreter(scoped / isolated) for each note' mode which you can se
## Make your own Interpreter
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
For your interpreter project, you need to make `interpreter-parent` as your parent project and use plugin `maven-enforcer-plugin`, `maven-dependency-plugin` and `maven-resources-plugin`. Here's one sample pom.xml
```
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
...
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
```
You should include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` as your interpreter's dependency in `pom.xml`. Bes
And you should put your jars under your interpreter directory with a specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter.
There are three locations where you can store your interpreter group, name and other information. Zeppelin server tries to find the location below. Next, Zeppelin tries to find `interpreter-setting.json` in your interpreter jar.

1
docs/index.md
View file

@ -142,6 +142,7 @@ limitations under the License.
* [Lens](./interpreter/lens.html)
* [Livy](./interpreter/livy.html)
* [markdown](./interpreter/markdown.html)
* [Neo4j](./interpreter/neo4j.html)
* [Pig](./interpreter/pig.html)
* [Postgresql, HAWQ](./interpreter/postgresql.html)
* [Python](./interpreter/python.html)

7
docs/interpreter/livy.md
View file

@ -144,7 +144,12 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
<td>zeppelin.livy.ssl.trustStorePassword</td>
<td></td>
<td>password for trustStore file. Used when livy ssl is enabled</td>
</tr>
</tr>
<tr>
<td>zeppelin.livy.http.headers</td>
<td>key_1: value_1; key_2: value_2</td>
<td>custom http headers when calling livy rest api. Each http header is separated by `;`, and each header is one key value pair where key value is separated by `:`</td>
</tr>
</table>
**We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.**

117
docs/interpreter/neo4j.md Normal file
View file

@ -0,0 +1,117 @@
---
layout: page
title: "Neo4j Interpreter for Apache Zeppelin"
description: "Neo4j is a native graph database, designed to store and process graphs from bottom to top."
group: interpreter
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Neo4j Interpreter for Apache Zeppelin
<div id="toc"></div>
## Overview
[Neo4j](https://neo4j.com/product/) is a native graph database, designed to store and process graphs from bottom to top.
![Neo4j - Interpreter - Video]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-interpreter-video.gif)
## Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
<th>Default</th>
<th>Description</th>
</tr>
<tr>
<td>neo4j.url</td>
<td>bolt://localhost:7687</td>
<td>The Neo4j's BOLT url.</td>
</tr>
<tr>
<td>neo4j.auth.type</td>
<td>BASIC</td>
<td>The Neo4j's authentication type (NONE, BASIC).</td>
</tr>
<tr>
<td>neo4j.auth.user</td>
<td>neo4j</td>
<td>The Neo4j user name.</td>
</tr>
<tr>
<td>neo4j.auth.password</td>
<td>neo4j</td>
<td>The Neo4j user password.</td>
</tr>
<tr>
<td>neo4j.max.concurrency</td>
<td>50</td>
<td>Max concurrency call from Zeppelin to Neo4j server.</td>
</tr>
</table>
<center>
![Interpreter configuration]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-config.png)
</center>
## Enabling the Neo4j Interpreter
In a notebook, to enable the **Neo4j** interpreter, click the **Gear** icon and select **Neo4j**.
## Using the Neo4j Interpreter
In a paragraph, use `%neo4j` to select the Neo4j interpreter and then input the Cypher commands.
For list of Cypher commands please refer to the official [Cyper Refcard](http://neo4j.com/docs/cypher-refcard/current/)
```bash
%neo4j
//Sample the TrumpWorld dataset
WITH
'https://docs.google.com/spreadsheets/u/1/d/1Z5Vo5pbvxKJ5XpfALZXvCzW26Cl4we3OaN73K9Ae5Ss/export?format=csv&gid=1996904412' AS url
LOAD CSV WITH HEADERS FROM url AS row
RETURN row.`Entity A`, row.`Entity A Type`, row.`Entity B`, row.`Entity B Type`, row.Connection, row.`Source(s)`
LIMIT 10
```
The Neo4j interpreter leverages the [Network display system](../usage/display_system/basic.html#network) allowing to visualize the them directly from the paragraph.
### Write your Cypher queries and navigate your graph
This query:
```bash
%neo4j
MATCH (vp:Person {name:"VLADIMIR PUTIN"}), (dt:Person {name:"DONALD J. TRUMP"})
MATCH path = allShortestPaths( (vp)-[*]-(dt) )
RETURN path
```
produces the following result_
![Neo4j - Graph - Result]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-graph.png)
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../usage/dynamic_form/intro.html) inside your queries. This query:
```bash
%neo4j
MATCH (o:Organization)-[r]-()
RETURN o.name, count(*), collect(distinct type(r)) AS types
ORDER BY count(*) DESC
LIMIT ${Show top=10}
```
produces the following result:
![Neo4j - Zeppelin - Dynamic Forms]({{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/neo4j-dynamic-forms.png)

8
docs/interpreter/spark.md
View file

@ -181,6 +181,7 @@ For example,
* **local[*]** in local mode
* **spark://master:7077** in standalone cluster
* **yarn-client** in Yarn client mode
* **yarn-cluster** in Yarn cluster mode
* **mesos://host:5050** in Mesos cluster
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way.
@ -188,6 +189,11 @@ For the further information about Spark & Zeppelin version compatibility, please
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
### 3. Yarn mode
Zeppelin support both yarn client and yarn cluster mode (yarn cluster mode is supported from 0.8.0). For yarn mode, you must specify `SPARK_HOME` & `HADOOP_CONF_DIR`.
You can either specify them in `zeppelin-env.sh`, or in interpreter setting page. Specifying them in `zeppelin-env.sh` means you can use only one version of `spark` & `hadoop`. Specifying them
in interpreter setting page means you can use multiple versions of `spark` & `hadoop` in one zeppelin instance.
## SparkContext, SQLContext, SparkSession, ZeppelinContext
SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.
Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x.
@ -424,7 +430,7 @@ It creates separated SparkContext per each notebook in `isolated` mode.
## IPython support
By default, zeppelin would use IPython in `pyspark` when IPython is available, Otherwise it would fall back to the original PySpark implementation.
If you don't want to use IPython, then you can set `zeppelin.spark.useIPython` as `false` in interpreter setting. For the IPython features, you can refer doc
If you don't want to use IPython, then you can set `zeppelin.pyspark.useIPython` as `false` in interpreter setting. For the IPython features, you can refer doc
[Python Interpreter](python.html)

34
docs/setup/operation/configuration.md
View file

@ -77,7 +77,19 @@ If both are defined, then the **environment variables** will take priority.
<td>*</td>
<td>Enables a way to specify a ',' separated list of allowed origins for REST and websockets. <br /> e.g. http://localhost:8080</td>
</tr>
<tr>
<tr>
<td><h6 class="properties">ZEPPELIN_CREDENTIALS_PERSIST</h6></td>
<td><h6 class="properties">zeppelin.credentials.persist</h6></td>
<td>true</td>
<td>Persist credentials on a JSON file (credentials.json)</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_CREDENTIALS_ENCRYPT_KEY</h6></td>
<td><h6 class="properties">zeppelin.credentials.encryptKey</h6></td>
<td></td>
<td>If provided, encrypt passwords on the credentials.json file (passwords will be stored as plain-text otherwise</td>
</tr>
<tr>
<td>N/A</td>
<td><h6 class="properties">zeppelin.anonymous.allowed</h6></td>
<td>true</td>
@ -203,6 +215,12 @@ If both are defined, then the **environment variables** will take priority.
<td>false</td>
<td>Save notebooks to S3 with server-side encryption enabled</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
<td>Optional override to control which signature algorithm should be used to sign AWS requests</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</h6></td>
<td><h6 class="properties">zeppelin.notebook.azure.connectionString</h6></td>
@ -411,6 +429,20 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order
</property>
```
### Storing user credentials
In order to avoid having to re-enter credentials every time you restart/redeploy Zeppelin, you can store the user credentials. Zeppelin supports this via the ZEPPELIN_CREDENTIALS_PERSIST configuration.
Please notice that passwords will be stored in *plain text* by default. To encrypt the passwords, use the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY config variable. This will encrypt passwords using the AES-128 algorithm.
You can generate an appropriate encryption key any way you'd like - for instance, by using the openssl tool:
```
openssl enc -aes-128-cbc -k secret -P -md sha1
```
*Important*: storing your encryption key in a configuration file is _not advised_. Depending on your environment security needs, you may want to consider utilizing a credentials server, storing the ZEPPELIN_CREDENTIALS_ENCRYPT_KEY as an OS env variable, or any other approach that would not colocate the encryption key and the encrypted content (the credentials.json file).
### Obfuscating Passwords using the Jetty Password Tool

12
docs/setup/storage/storage.md
View file

@ -30,7 +30,7 @@ There are few notebook storage systems available for a use out of the box:
* (default) use local file system and version it using local Git repository - `GitNotebookRepo`
* all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
* all notes are saved in the notebook folder in hdfs - `HdfsNotebookRepo`
* all notes are saved in the notebook folder in hadoop compatible file system - `FileSystemNotebookRepo`
* storage using Amazon S3 service - `S3NotebookRepo`
* storage using Azure service - `AzureNotebookRepo`
* storage using MongoDB - `MongoNotebookRepo`
@ -54,16 +54,16 @@ To enable versioning for all your local notebooks though a standard Git reposito
</br>
## Notebook Storage in Hdfs repository <a name="Hdfs"></a>
## Notebook Storage in hadoop compatible file system repository <a name="Hdfs"></a>
Notes may be stored in hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `HdfsNotebookRepo`, then `zeppelin.notebook.dir` is the path on hdfs. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files.
If your hadoop cluster is kerberized, then you need to specify `zeppelin.hdfs.keytab` and `zeppelin.hdfs.principal`
Notes may be stored in hadoop compatible file system such as hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `FileSystemNotebookRepo`, then `zeppelin.notebook.dir` is the path on the hadoop compatible file system. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files.
If your hadoop cluster is kerberized, then you need to specify `zeppelin.server.kerberos.keytab` and `zeppelin.server.kerberos.principal`
```
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
<description>hdfs notebook persistence layer implementation</description>
<value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
<description>hadoop compatible file system notebook persistence layer implementation</description>
</property>
```

53
docs/usage/rest_api/notebook.md
View file

@ -752,6 +752,59 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete,
</tr>
</table>
<br/>
### Update paragraph
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```PUT``` method update paragraph contents using given id, e.g. <code>{"text": "hello"}</code>
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/paragraph/[paragraphId]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td>Bad Request code</td>
<td>400</td>
</tr>
<tr>
<td>Forbidden code</td>
<td>403</td>
</tr>
<tr>
<td>Not Found code</td>
<td>404</td>
</tr>
<tr>
<td>Fail code</td>
<td>500</td>
</tr>
<tr>
<td>sample JSON input</td>
<td><pre>
{
"title": "Hello world",
"text": "println(\"hello world\")"
}</pre></td>
</tr>
<tr>
<td>sample JSON response</td>
<td><pre>
{
"status": "OK",
"message": ""
}
}</pre></td>
</tr>
</table>
<br/>
### Update paragraph configuration
<table class="table-configuration">

53
elasticsearch/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<artifactId>zeppelin-elasticsearch</artifactId>
@ -32,6 +32,7 @@
<name>Zeppelin: Elasticsearch interpreter</name>
<properties>
<interpreter.name>elasticsearch</interpreter.name>
<elasticsearch.version>2.4.3</elasticsearch.version>
<httpasyncclient.version>4.0.2</httpasyncclient.version>
<guava.version>18.0</guava.version>
@ -93,54 +94,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/elasticsearch</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/elasticsearch</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

8
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java
View file

@ -112,7 +112,7 @@ public class ElasticsearchInterpreter extends Interpreter {
@Override
public void open() {
logger.info("Properties: {}", getProperty());
logger.info("Properties: {}", getProperties());
String clientType = getProperty(ELASTICSEARCH_CLIENT_TYPE);
clientType = clientType == null ? null : clientType.toLowerCase();
@ -123,15 +123,15 @@ public class ElasticsearchInterpreter extends Interpreter {
catch (final NumberFormatException e) {
this.resultSize = 10;
logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " +
property.get(ELASTICSEARCH_RESULT_SIZE), e);
getProperty(ELASTICSEARCH_RESULT_SIZE), e);
}
try {
if (StringUtils.isEmpty(clientType) || "transport".equals(clientType)) {
elsClient = new TransportBasedClient(getProperty());
elsClient = new TransportBasedClient(getProperties());
}
else if ("http".equals(clientType)) {
elsClient = new HttpBasedClient(getProperty());
elsClient = new HttpBasedClient(getProperties());
}
else {
logger.error("Unknown type of Elasticsearch client: " + clientType);

59
file/pom.xml
View file

@ -20,9 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,11 +34,13 @@
<properties>
<!--library versions-->
<interpreter.name>file</interpreter.name>
<ws.rsapi.version>2.0</ws.rsapi.version>
<jersey.common.version>2.22.2</jersey.common.version>
<!--plugin versions-->
<plugin.surefire.version>2.18.1</plugin.surefire.version>
<interpreter.name>file</interpreter.name>
</properties>
<dependencies>
@ -79,62 +82,14 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${plugin.surefire.version}</version>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/file</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/file</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<!--<includeScope>runtime</includeScope>-->
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

3
file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java
View file

@ -20,6 +20,7 @@ package org.apache.zeppelin.file;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
@ -86,7 +87,7 @@ public abstract class FileInterpreter extends Interpreter {
// Functions that each file system implementation must override
public abstract String listAll(String path);
public abstract String listAll(String path) throws InterpreterException;
public abstract boolean isDirectory(String path);

2
file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java
View file

@ -202,7 +202,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
return "No such File or directory";
}
public String listAll(String path) {
public String listAll(String path) throws InterpreterException {
String all = "";
if (exceptionOnConnect != null)
return "Error connecting to provided endpoint.";

75
flink/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -35,6 +35,7 @@
<properties>
<!--library versions-->
<interpreter.name>flink</interpreter.name>
<flink.version>1.1.3</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<scala.macros.version>2.0.1</scala.macros.version>
@ -280,68 +281,16 @@
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

8
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -17,7 +17,6 @@
*/
package org.apache.zeppelin.flink;
import java.lang.reflect.InvocationTargetException;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -34,10 +33,8 @@ import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
@ -46,11 +43,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Console;
import scala.None;
import scala.Option;
import scala.Some;
import scala.collection.JavaConversions;
import scala.collection.immutable.Nil;
import scala.concurrent.duration.FiniteDuration;
import scala.runtime.AbstractFunction0;
import scala.tools.nsc.Settings;
@ -80,7 +74,7 @@ public class FlinkInterpreter extends Interpreter {
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
Properties intpProperty = getProperties();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));

52
geode/pom.xml
View file

@ -21,9 +21,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -34,6 +35,7 @@
<properties>
<!--library versions-->
<interpreter.name>geode</interpreter.name>
<geode.version>1.1.0</geode.version>
<commons.exec.version>1.3</commons.exec.version>
</properties>
@ -86,54 +88,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

81
groovy/pom.xml
View file

@ -20,11 +20,11 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<!--version>0.6.2</version-->
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,6 +33,10 @@
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Groovy interpreter</name>
<properties>
<interpreter.name>groovy</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
@ -67,81 +71,14 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<showDeprecation>true</showDeprecation>
<compilerArgs>
<!--arg>-verbose</arg-->
<arg>-Xlint:unchecked</arg>
</compilerArgs>
</configuration>
</plugin>
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
<executions>
</executions>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

8
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
View file

@ -172,7 +172,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
/**
* starts or continues rendering html/angular and returns MarkupBuilder to build html.
* <pre> g.html().with{
* h1("hello")
* h1("hello")
* h2("world")
* }</pre>
*/
@ -316,12 +316,12 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
@ZeppelinApi
public void run(String noteId, String paragraphId, InterpreterContext context) {
if (paragraphId.equals(context.getParagraphId())) {
throw new InterpreterException("Can not run current Paragraph");
throw new RuntimeException("Can not run current Paragraph");
}
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, paragraphId,
context);
if (runners.size() <= 0) {
throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size());
throw new RuntimeException("Paragraph " + paragraphId + " not found " + runners.size());
}
for (InterpreterContextRunner r : runners) {
r.run();
@ -338,7 +338,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
if (runners.size() <= 0) {
throw new InterpreterException("Note " + noteId + " not found " + runners.size());
throw new RuntimeException("Note " + noteId + " not found " + runners.size());
}
for (InterpreterContextRunner r : runners) {

9
groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java
View file

@ -17,8 +17,6 @@
package org.apache.zeppelin.groovy;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.io.PrintWriter;
import java.io.File;
@ -26,10 +24,8 @@ import java.util.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
@ -40,7 +36,6 @@ import org.slf4j.LoggerFactory;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.runtime.ResourceGroovyMethods;
import org.codehaus.groovy.runtime.StackTraceUtils;
import java.util.concurrent.ConcurrentHashMap;
@ -167,7 +162,7 @@ public class GroovyInterpreter extends Interpreter {
//put shared bindings evaluated in this interpreter
bindings.putAll(sharedBindings);
//put predefined bindings
bindings.put("g", new GObject(log, out, this.getProperty(), contextInterpreter, bindings));
bindings.put("g", new GObject(log, out, this.getProperties(), contextInterpreter, bindings));
bindings.put("out", new PrintWriter(out, true));
script.run();
@ -204,7 +199,7 @@ public class GroovyInterpreter extends Interpreter {
Thread t = (Thread) object;
t.dumpStack();
t.interrupt();
//t.stop(); //TODO: need some way to terminate maybe through GObject..
//t.stop(); //TODO(dlukyanov): need some way to terminate maybe through GObject..
} catch (Throwable t) {
log.error("Failed to cancel script: " + t, t);
}

53
hbase/pom.xml
View file

@ -20,9 +20,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,6 +34,7 @@
<properties>
<!--library versions-->
<interpreter.name>hbase</interpreter.name>
<hbase.hbase.version>1.0.0</hbase.hbase.version>
<hbase.hadoop.version>2.6.0</hbase.hadoop.version>
<jruby.version>1.6.8</jruby.version>
@ -115,55 +117,14 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

4
hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
View file

@ -68,7 +68,7 @@ public class HbaseInterpreter extends Interpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON);
this.writer = new StringWriter();
scriptingContainer.setOutput(this.writer);
@ -88,7 +88,7 @@ public class HbaseInterpreter extends Interpreter {
}
logger.info("Absolute Ruby Source:" + abs_ruby_src.toString());
// hirb.rb:41 requires the following system property to be set.
// hirb.rb:41 requires the following system properties to be set.
Properties sysProps = System.getProperties();
sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString());

3
hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
View file

@ -15,6 +15,7 @@
package org.apache.zeppelin.hbase;
import org.apache.log4j.BasicConfigurator;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.BeforeClass;
import org.junit.Test;
@ -35,7 +36,7 @@ public class HbaseInterpreterTest {
private static HbaseInterpreter hbaseInterpreter;
@BeforeClass
public static void setUp() throws NullPointerException {
public static void setUp() throws NullPointerException, InterpreterException {
BasicConfigurator.configure();
Properties properties = new Properties();
properties.put("hbase.home", "");

51
helium-dev/pom.xml
View file

@ -23,8 +23,9 @@
<parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -32,6 +33,10 @@
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Helium development interpreter</name>
<properties>
<interpreter.name>helium-dev</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
@ -43,46 +48,14 @@
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/helium-dev</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/helium-dev</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

3
helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java
View file

@ -74,7 +74,8 @@ public class DevInterpreter extends Interpreter {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
this.context = context;
try {
return interpreterEvent.interpret(st, context);

56
ignite/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<artifactId>zeppelin-ignite_2.10</artifactId>
@ -32,7 +32,8 @@
<name>Zeppelin: Apache Ignite interpreter</name>
<properties>
<ignite.version>2.1.0</ignite.version>
<interpreter.name>ignite</interpreter.name>
<ignite.version>2.2.0</ignite.version>
</properties>
<dependencies>
@ -106,55 +107,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

2
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java
View file

@ -93,7 +93,7 @@ public class IgniteSqlInterpreter extends Interpreter {
}
@Override
public void close() {
public void close() throws InterpreterException {
try {
if (conn != null) {
conn.close();

3
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -27,6 +27,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
@ -82,7 +83,7 @@ public class IgniteSqlInterpreterTest {
}
@After
public void tearDown() {
public void tearDown() throws InterpreterException {
intp.close();
ignite.close();
}

138
interpreter-parent/pom.xml Normal file
View file

@ -0,0 +1,138 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>interpreter-parent</artifactId>
<packaging>pom</packaging>
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Interpreter Parent</name>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>0.8.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-interpreter-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-interpreter-setting</id>
<phase>package</phase>
<goals>
<goal>resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/${interpreter.name}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>

52
jdbc/pom.xml
View file

@ -21,9 +21,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -34,6 +35,7 @@
<properties>
<!--library versions-->
<interpreter.name>jdbc</interpreter.name>
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
<hadoop.common.version>2.7.2</hadoop.common.version>
<h2.version>1.4.190</h2.version>
@ -158,54 +160,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

85
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -38,14 +38,13 @@ import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -171,7 +170,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
@Override
public void open() {
super.open();
for (String propertyKey : property.stringPropertyNames()) {
for (String propertyKey : properties.stringPropertyNames()) {
logger.debug("propertyKey: {}", propertyKey);
String[] keyValue = propertyKey.split("\\.", 2);
if (2 == keyValue.length) {
@ -184,7 +183,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
prefixProperties = new Properties();
basePropretiesMap.put(keyValue[0].trim(), prefixProperties);
}
prefixProperties.put(keyValue[1].trim(), property.getProperty(propertyKey));
prefixProperties.put(keyValue[1].trim(), getProperty(propertyKey));
}
}
@ -210,8 +209,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
protected boolean isKerboseEnabled() {
if (!isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
if (!isEmpty(getProperty("zeppelin.jdbc.auth.type"))) {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(properties);
if (authType.equals(KERBEROS)) {
return true;
}
@ -278,11 +277,17 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
private void initConnectionPoolMap() {
for (JDBCUserConfigurations configurations : jdbcUserConfigurationsMap.values()) {
for (String key : jdbcUserConfigurationsMap.keySet()) {
try {
closeDBPool(key, DEFAULT_KEY);
} catch (SQLException e) {
logger.error("Error while closing database pool.", e);
}
try {
JDBCUserConfigurations configurations = jdbcUserConfigurationsMap.get(key);
configurations.initConnectionPoolMap();
} catch (Exception e) {
logger.error("Error while closing initConnectionPoolMap...", e);
} catch (SQLException e) {
logger.error("Error while closing initConnectionPoolMap.", e);
}
}
}
@ -349,7 +354,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
private void setUserProperty(String propertyKey, InterpreterContext interpreterContext)
throws SQLException, IOException {
throws SQLException, IOException, InterpreterException {
String user = interpreterContext.getAuthenticationInfo().getUser();
@ -417,18 +422,19 @@ public class JDBCInterpreter extends KerberosInterpreter {
final Properties properties = jdbcUserConfigurations.getPropertyMap(propertyKey);
final String url = properties.getProperty(URL_KEY);
if (isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
if (isEmpty(getProperty("zeppelin.jdbc.auth.type"))) {
connection = getConnectionFromPool(url, user, propertyKey, properties);
} else {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
UserGroupInformation.AuthenticationMethod authType =
JDBCSecurityImpl.getAuthtype(getProperties());
final String connectionUrl = appendProxyUserToURL(url, user, propertyKey);
JDBCSecurityImpl.createSecureConfiguration(property, authType);
JDBCSecurityImpl.createSecureConfiguration(getProperties(), authType);
switch (authType) {
case KERBEROS:
if (user == null || "false".equalsIgnoreCase(
property.getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties);
} else {
if (basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) {
@ -490,7 +496,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
return connectionUrl.toString();
}
private String getPassword(Properties properties) throws IOException {
private String getPassword(Properties properties) throws IOException, InterpreterException {
if (isNotEmpty(properties.getProperty(PASSWORD_KEY))) {
return properties.getProperty(PASSWORD_KEY);
} else if (isNotEmpty(properties.getProperty(JDBC_JCEKS_FILE))
@ -518,7 +524,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
return null;
}
private String getResults(ResultSet resultSet, boolean isTableType)
private String getResults(ResultSet resultSet, boolean isTableType, MutableBoolean isComplete)
throws SQLException {
ResultSetMetaData md = resultSet.getMetaData();
StringBuilder msg;
@ -537,7 +543,11 @@ public class JDBCInterpreter extends KerberosInterpreter {
msg.append(NEWLINE);
int displayRowCount = 0;
while (displayRowCount < getMaxResult() && resultSet.next()) {
while (resultSet.next()) {
if (displayRowCount >= getMaxResult()) {
isComplete.setValue(false);
break;
}
for (int i = 1; i < md.getColumnCount() + 1; i++) {
Object resultObject;
String resultValue;
@ -653,7 +663,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
Connection connection;
Connection connection = null;
Statement statement;
ResultSet resultSet = null;
String paragraphId = interpreterContext.getParagraphId();
@ -668,11 +678,21 @@ public class JDBCInterpreter extends KerberosInterpreter {
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
try {
connection = getConnection(propertyKey, interpreterContext);
if (connection == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
} catch (Exception e) {
String errorMsg = Throwables.getStackTraceAsString(e);
try {
closeDBPool(user, propertyKey);
} catch (SQLException e1) {
logger.error("Cannot close DBPool for user, propertyKey: " + user + propertyKey, e1);
}
interpreterResult.add(errorMsg);
return new InterpreterResult(Code.ERROR, interpreterResult.message());
}
if (connection == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
try {
List<String> sqlArray;
if (splitQuery) {
sqlArray = splitSqlQueries(sql);
@ -706,10 +726,11 @@ public class JDBCInterpreter extends KerberosInterpreter {
interpreterResult.add(InterpreterResult.Type.TEXT,
"Query executed successfully.");
} else {
MutableBoolean isComplete = new MutableBoolean(true);
String results = getResults(resultSet,
!containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE));
!containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE), isComplete);
interpreterResult.add(results);
if (resultSet.next()) {
if (!isComplete.booleanValue()) {
interpreterResult.add(ResultMessages.getExceedsLimitRowsMessage(getMaxResult(),
String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY)));
}
@ -734,6 +755,12 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
}
}
} catch (Throwable e) {
logger.error("Cannot run " + sql, e);
String errorMsg = Throwables.getStackTraceAsString(e);
interpreterResult.add(errorMsg);
return new InterpreterResult(Code.ERROR, interpreterResult.message());
} finally {
//In case user ran an insert/update/upsert statement
if (connection != null) {
try {
@ -744,16 +771,6 @@ public class JDBCInterpreter extends KerberosInterpreter {
} catch (SQLException e) { /*ignored*/ }
}
getJDBCConfiguration(user).removeStatement(paragraphId);
} catch (Throwable e) {
logger.error("Cannot run " + sql, e);
String errorMsg = Throwables.getStackTraceAsString(e);
try {
closeDBPool(user, propertyKey);
} catch (SQLException e1) {
logger.error("Cannot close DBPool for user, propertyKey: " + user + propertyKey, e1);
}
interpreterResult.add(errorMsg);
return new InterpreterResult(Code.ERROR, interpreterResult.message());
}
return interpreterResult;
}
@ -832,7 +849,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
InterpreterContext interpreterContext) throws InterpreterException {
List<InterpreterCompletion> candidates = new ArrayList<>();
String propertyKey = getPropertyKey(buf);
String sqlCompleterKey =

6
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java
View file

@ -48,12 +48,6 @@ public class JDBCUserConfigurations {
}
public void initConnectionPoolMap() throws SQLException {
Iterator<String> it = poolingDriverMap.keySet().iterator();
while (it.hasNext()) {
String driverName = it.next();
poolingDriverMap.get(driverName).closePool(driverName);
it.remove();
}
poolingDriverMap.clear();
isSuccessful.clear();
}

3
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -37,6 +37,7 @@ import java.util.Properties;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.FIFOScheduler;
@ -349,7 +350,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
}
@Test
public void testAutoCompletion() throws SQLException, IOException {
public void testAutoCompletion() throws SQLException, IOException, InterpreterException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");

56
kylin/pom.xml
View file

@ -21,9 +21,10 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -33,6 +34,10 @@
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Kylin interpreter</name>
<properties>
<interpreter.name>kylin</interpreter.name>
</properties>
<dependencies>
<dependency>
@ -61,57 +66,14 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

67
lens/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -34,6 +34,7 @@
<properties>
<!--library versions-->
<interpreter.name>lens</interpreter.name>
<lens.version>2.5.0-beta</lens.version>
<spring-shell.version>1.1.0.RELEASE</spring-shell.version>
<hadoop-common.version>2.4.0</hadoop-common.version>
@ -163,69 +164,13 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
<filesets>
<fileset>
<directory>${basedir}/../interpreter/lens</directory>
<followSymlinks>false</followSymlinks>
</fileset>
</filesets>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

55
livy/pom.xml
View file

@ -22,10 +22,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -36,6 +36,7 @@
<properties>
<!--library versions-->
<interpreter.name>livy</interpreter.name>
<commons.exec.version>1.3</commons.exec.version>
<spring.web.version>4.3.0.RELEASE</spring.web.version>
<spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
@ -193,56 +194,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>

48
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
View file

@ -41,6 +41,7 @@ import org.apache.zeppelin.interpreter.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.MediaType;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
@ -61,6 +62,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@ -79,6 +82,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
protected boolean displayAppInfo;
protected LivyVersion livyVersion;
private RestTemplate restTemplate;
private Map<String, String> customHeaders = new HashMap<>();
Set<Object> paragraphsToCancel = Collections.newSetFromMap(
new ConcurrentHashMap<Object, Boolean>());
@ -95,12 +99,39 @@ public abstract class BaseLivyInterpreter extends Interpreter {
this.pullStatusInterval = Integer.parseInt(
property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
this.restTemplate = createRestTemplate();
if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) {
String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";");
for (String header : headers) {
String[] splits = header.split(":", -1);
if (splits.length != 2) {
throw new RuntimeException("Invalid format of http headers: " + header +
", valid http header format is HEADER_NAME:HEADER_VALUE");
}
customHeaders.put(splits[0].trim(), envSubstitute(splits[1].trim()));
}
}
}
private String envSubstitute(String value) {
String newValue = new String(value);
Pattern pattern = Pattern.compile("\\$\\{(.*)\\}");
Matcher matcher = pattern.matcher(value);
while (matcher.find()) {
String env = matcher.group(1);
newValue = newValue.replace("${" + env + "}", System.getenv(env));
}
return newValue;
}
// only for testing
Map<String, String> getCustomHeaders() {
return customHeaders;
}
public abstract String getSessionKind();
@Override
public void open() {
public void open() throws InterpreterException {
try {
initLivySession();
} catch (LivyException e) {
@ -197,7 +228,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
throws LivyException {
try {
Map<String, String> conf = new HashMap<>();
for (Map.Entry<Object, Object> entry : property.entrySet()) {
for (Map.Entry<Object, Object> entry : getProperties().entrySet()) {
if (entry.getKey().toString().startsWith("livy.spark.") &&
!entry.getValue().toString().isEmpty())
conf.put(entry.getKey().toString().substring(5), entry.getValue().toString());
@ -427,15 +458,15 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private RestTemplate createRestTemplate() {
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
String keytabLocation = getProperty("zeppelin.livy.keytab");
String principal = getProperty("zeppelin.livy.principal");
boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
StringUtils.isNotEmpty(principal);
HttpClient httpClient = null;
if (livyURL.startsWith("https:")) {
String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword");
String keystoreFile = getProperty("zeppelin.livy.ssl.trustStore");
String password = getProperty("zeppelin.livy.ssl.trustStorePassword");
if (StringUtils.isBlank(keystoreFile)) {
throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl");
}
@ -520,8 +551,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
targetURL = livyURL + targetURL;
LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData);
HttpHeaders headers = new HttpHeaders();
headers.add("Content-Type", "application/json");
headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE);
headers.add("X-Requested-By", "zeppelin");
for (Map.Entry<String, String> entry : customHeaders.entrySet()) {
headers.add(entry.getKey(), entry.getValue());
}
ResponseEntity<String> response = null;
try {
if (method.equals("POST")) {

4
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -59,7 +59,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
this.sparkInterpreter = getSparkInterpreter();
// As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession
// to judge whether it is using spark2.
@ -93,7 +93,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
}
}
private LivySparkInterpreter getSparkInterpreter() {
private LivySparkInterpreter getSparkInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
LivySparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName());

34
livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
View file

@ -28,6 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Properties;
@ -74,8 +75,9 @@ public class LivyInterpreterIT {
return true;
}
// @Test
public void testSparkInterpreterRDD() {
public void testSparkInterpreterRDD() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -194,8 +196,9 @@ public class LivyInterpreterIT {
}
}
// @Test
public void testSparkInterpreterDataFrame() {
public void testSparkInterpreterDataFrame() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -283,7 +286,7 @@ public class LivyInterpreterIT {
}
// @Test
public void testSparkSQLInterpreter() {
public void testSparkSQLInterpreter() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -318,7 +321,7 @@ public class LivyInterpreterIT {
// @Test
public void testSparkSQLCancellation() {
public void testSparkSQLCancellation() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -399,7 +402,7 @@ public class LivyInterpreterIT {
}
// @Test
public void testStringWithTruncation() {
public void testStringWithTruncation() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -458,8 +461,9 @@ public class LivyInterpreterIT {
}
}
// @Test
public void testStringWithoutTruncation() {
public void testStringWithoutTruncation() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -524,7 +528,7 @@ public class LivyInterpreterIT {
}
@Test
public void testPySparkInterpreter() throws LivyException {
public void testPySparkInterpreter() throws LivyException, InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -555,6 +559,16 @@ public class LivyInterpreterIT {
assertTrue(result.message().get(0).getData().contains("Traceback"));
}
// test utf-8 Encoding
try {
String utf8Str = "你你你你你你好";
InterpreterResult result = pysparkInterpreter.interpret("print(\""+utf8Str+"\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains(utf8Str));
}catch (Exception e) {
e.printStackTrace();
}
try {
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@ -634,7 +648,7 @@ public class LivyInterpreterIT {
}
// @Test
public void testSparkInterpreterWithDisplayAppInfo() {
public void testSparkInterpreterWithDisplayAppInfo() throws InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -673,7 +687,7 @@ public class LivyInterpreterIT {
}
// @Test
public void testSparkRInterpreter() throws LivyException {
public void testSparkRInterpreter() throws LivyException, InterpreterException {
if (!checkPreCondition()) {
return;
}
@ -745,7 +759,7 @@ public class LivyInterpreterIT {
}
// @Test
public void testLivyTutorialNote() throws IOException {
public void testLivyTutorialNote() throws IOException, InterpreterException {
if (!checkPreCondition()) {
return;
}

8
livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
View file

@ -39,9 +39,17 @@ public class LivySQLInterpreterTest {
properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
properties.setProperty("zeppelin.livy.session.create_timeout", "120");
properties.setProperty("zeppelin.livy.spark.sql.maxResult", "3");
properties.setProperty("zeppelin.livy.http.headers", "HEADER_1: VALUE_1_${HOME}");
sqlInterpreter = new LivySparkSQLInterpreter(properties);
}
@Test
public void testHttpHeaders() {
assertEquals(1, sqlInterpreter.getCustomHeaders().size());
assertTrue(sqlInterpreter.getCustomHeaders().get("HEADER_1").startsWith("VALUE_1_"));
assertNotEquals("VALUE_1_${HOME}", sqlInterpreter.getCustomHeaders().get("HEADER_1"));
}
@Test
public void testParseSQLOutput() {
// Empty sql output

53
markdown/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,6 +33,7 @@
<name>Zeppelin: Markdown interpreter</name>
<properties>
<interpreter.name>md</interpreter.name>
<commons.lang3.version>3.4</commons.lang3.version>
<markdown4j.version>2.2-cj-1.0</markdown4j.version>
<pegdown.version>1.6.0</pegdown.version>
@ -85,54 +86,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/md</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/md</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

144
neo4j/pom.xml Normal file
View file

@ -0,0 +1,144 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-neo4j</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Neo4j interpreter</name>
<properties>
<neo4j.driver.version>1.4.3</neo4j.driver.version>
<test.neo4j.kernel.version>3.2.3</test.neo4j.kernel.version>
<neo4j.version>3.2.3</neo4j.version>
<jackson.version>2.8.9</jackson.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.neo4j.driver</groupId>
<artifactId>neo4j-java-driver</artifactId>
<version>${neo4j.driver.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.neo4j.test</groupId>
<artifactId>neo4j-harness</artifactId>
<version>${neo4j.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/neo4j</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/neo4j</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

151
neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jConnectionManager.java Normal file
View file

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.graph.neo4j;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import org.neo4j.driver.v1.AuthToken;
import org.neo4j.driver.v1.AuthTokens;
import org.neo4j.driver.v1.Config;
import org.neo4j.driver.v1.Driver;
import org.neo4j.driver.v1.GraphDatabase;
import org.neo4j.driver.v1.Session;
import org.neo4j.driver.v1.StatementResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Neo4j connection manager for Zeppelin.
*/
public class Neo4jConnectionManager {
static final Logger LOGGER = LoggerFactory.getLogger(Neo4jConnectionManager.class);
public static final String NEO4J_SERVER_URL = "neo4j.url";
public static final String NEO4J_AUTH_TYPE = "neo4j.auth.type";
public static final String NEO4J_AUTH_USER = "neo4j.auth.user";
public static final String NEO4J_AUTH_PASSWORD = "neo4j.auth.password";
public static final String NEO4J_MAX_CONCURRENCY = "neo4j.max.concurrency";
private static final Pattern PROPERTY_PATTERN = Pattern.compile("\\{\\w+\\}");
private static final String REPLACE_CURLY_BRACKETS = "\\{|\\}";
private static final Pattern $_PATTERN = Pattern.compile("\\$\\w+\\}");
private static final String REPLACE_$ = "\\$";
private Driver driver = null;
private final String neo4jUrl;
private final Config config;
private final AuthToken authToken;
/**
*
* Enum type for the AuthToken
*
*/
public enum Neo4jAuthType {NONE, BASIC}
public Neo4jConnectionManager(Properties properties) {
this.neo4jUrl = properties.getProperty(NEO4J_SERVER_URL);
this.config = Config.build()
.withMaxIdleSessions(Integer.parseInt(properties.getProperty(NEO4J_MAX_CONCURRENCY)))
.toConfig();
String authType = properties.getProperty(NEO4J_AUTH_TYPE);
switch (Neo4jAuthType.valueOf(authType.toUpperCase())) {
case BASIC:
String username = properties.getProperty(NEO4J_AUTH_USER);
String password = properties.getProperty(NEO4J_AUTH_PASSWORD);
this.authToken = AuthTokens.basic(username, password);
break;
case NONE:
LOGGER.debug("Creating NONE authentication");
this.authToken = AuthTokens.none();
break;
default:
throw new RuntimeException("Neo4j authentication type not supported");
}
}
private Driver getDriver() {
if (driver == null) {
driver = GraphDatabase.driver(this.neo4jUrl, this.authToken, this.config);
}
return driver;
}
public void open() {
getDriver();
}
public void close() {
getDriver().close();
}
private Session getSession() {
return getDriver().session();
}
public StatementResult execute(String cypherQuery,
InterpreterContext interpreterContext) {
Map<String, Object> params = new HashMap<>();
if (interpreterContext != null) {
ResourcePool resourcePool = interpreterContext.getResourcePool();
Set<String> keys = extractParams(cypherQuery, PROPERTY_PATTERN, REPLACE_CURLY_BRACKETS);
keys.addAll(extractParams(cypherQuery, $_PATTERN, REPLACE_$));
for (String key : keys) {
Resource resource = resourcePool.get(key);
if (resource != null) {
params.put(key, resource.get());
}
}
}
LOGGER.debug("Executing cypher query {} with params {}", cypherQuery, params);
StatementResult result;
try (Session session = getSession()) {
result = params.isEmpty()
? getSession().run(cypherQuery) : getSession().run(cypherQuery, params);
}
return result;
}
public StatementResult execute(String cypherQuery) {
return execute(cypherQuery, null);
}
private Set<String> extractParams(String cypherQuery, Pattern pattern, String replaceChar) {
Matcher matcher = pattern.matcher(cypherQuery);
Set<String> keys = new HashSet<>();
while (matcher.find()) {
keys.add(matcher.group().replaceAll(replaceChar, StringUtils.EMPTY));
}
return keys;
}
}

274
neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreter.java Normal file
View file

@ -0,0 +1,274 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.graph.neo4j;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.graph.neo4j.utils.Neo4jConversionUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.graph.GraphResult;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.neo4j.driver.internal.types.InternalTypeSystem;
import org.neo4j.driver.internal.util.Iterables;
import org.neo4j.driver.v1.Record;
import org.neo4j.driver.v1.StatementResult;
import org.neo4j.driver.v1.Value;
import org.neo4j.driver.v1.types.Node;
import org.neo4j.driver.v1.types.Relationship;
import org.neo4j.driver.v1.types.TypeSystem;
import org.neo4j.driver.v1.util.Pair;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Neo4j interpreter for Zeppelin.
*/
public class Neo4jCypherInterpreter extends Interpreter {
private static final String TABLE = "%table";
public static final String NEW_LINE = "\n";
public static final String TAB = "\t";
private static final String MAP_KEY_TEMPLATE = "%s.%s";
private Map<String, String> labels;
private Set<String> types;
private final Neo4jConnectionManager neo4jConnectionManager;
private final ObjectMapper jsonMapper = new ObjectMapper();
public Neo4jCypherInterpreter(Properties properties) {
super(properties);
this.neo4jConnectionManager = new Neo4jConnectionManager(properties);
}
@Override
public void open() {
this.neo4jConnectionManager.open();
}
@Override
public void close() {
this.neo4jConnectionManager.close();
}
public Map<String, String> getLabels(boolean refresh) {
if (labels == null || refresh) {
Map<String, String> old = labels == null ?
new LinkedHashMap<String, String>() : new LinkedHashMap<>(labels);
labels = new LinkedHashMap<>();
StatementResult result = this.neo4jConnectionManager.execute("CALL db.labels()");
Set<String> colors = new HashSet<>();
while (result.hasNext()) {
Record record = result.next();
String label = record.get("label").asString();
String color = old.get(label);
while (color == null || colors.contains(color)) {
color = Neo4jConversionUtils.getRandomLabelColor();
}
colors.add(color);
labels.put(label, color);
}
}
return labels;
}
private Set<String> getTypes(boolean refresh) {
if (types == null || refresh) {
types = new HashSet<>();
StatementResult result = this.neo4jConnectionManager.execute("CALL db.relationshipTypes()");
while (result.hasNext()) {
Record record = result.next();
types.add(record.get("relationshipType").asString());
}
}
return types;
}
@Override
public InterpreterResult interpret(String cypherQuery, InterpreterContext interpreterContext) {
logger.info("Opening session");
if (StringUtils.isBlank(cypherQuery)) {
return new InterpreterResult(Code.SUCCESS);
}
try {
StatementResult result = this.neo4jConnectionManager.execute(cypherQuery,
interpreterContext);
Set<Node> nodes = new HashSet<>();
Set<Relationship> relationships = new HashSet<>();
List<String> columns = new ArrayList<>();
List<List<String>> lines = new ArrayList<List<String>>();
while (result.hasNext()) {
Record record = result.next();
List<Pair<String, Value>> fields = record.fields();
List<String> line = new ArrayList<>();
for (Pair<String, Value> field : fields) {
if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.NODE())) {
nodes.add(field.value().asNode());
} else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.RELATIONSHIP())) {
relationships.add(field.value().asRelationship());
} else if (field.value().hasType(InternalTypeSystem.TYPE_SYSTEM.PATH())) {
nodes.addAll(Iterables.asList(field.value().asPath().nodes()));
relationships.addAll(Iterables.asList(field.value().asPath().relationships()));
} else {
setTabularResult(field.key(), field.value(), columns, line,
InternalTypeSystem.TYPE_SYSTEM);
}
}
if (!line.isEmpty()) {
lines.add(line);
}
}
if (!nodes.isEmpty()) {
return renderGraph(nodes, relationships);
} else {
return renderTable(columns, lines);
}
} catch (Exception e) {
logger.error("Exception while interpreting cypher query", e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
private void setTabularResult(String key, Object obj, List<String> columns, List<String> line,
TypeSystem typeSystem) {
if (obj instanceof Value) {
Value value = (Value) obj;
if (value.hasType(typeSystem.MAP())) {
Map<String, Object> map = value.asMap();
for (Entry<String, Object> entry : map.entrySet()) {
setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(),
columns, line, typeSystem);
}
} else {
addValueToLine(key, columns, line, value);
}
} else if (obj instanceof Map) {
Map<String, Object> map = (Map<String, Object>) obj;
for (Entry<String, Object> entry : map.entrySet()) {
setTabularResult(String.format(MAP_KEY_TEMPLATE, key, entry.getKey()), entry.getValue(),
columns, line, typeSystem);
}
} else {
addValueToLine(key, columns, line, obj);
}
}
private void addValueToLine(String key, List<String> columns, List<String> line, Object value) {
if (!columns.contains(key)) {
columns.add(key);
}
int position = columns.indexOf(key);
if (line.size() < columns.size()) {
for (int i = line.size(); i < columns.size(); i++) {
line.add(null);
}
}
if (value != null) {
if (value instanceof Value) {
Value val = (Value) value;
if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.LIST())) {
value = val.asList();
} else if (val.hasType(InternalTypeSystem.TYPE_SYSTEM.MAP())) {
value = val.asMap();
}
}
if (value instanceof Collection) {
try {
value = jsonMapper.writer().writeValueAsString(value);
} catch (Exception ignored) {}
}
}
line.set(position, value == null ? null : value.toString());
}
private InterpreterResult renderTable(List<String> cols, List<List<String>> lines) {
logger.info("Executing renderTable method");
StringBuilder msg = null;
if (cols.isEmpty()) {
msg = new StringBuilder();
} else {
msg = new StringBuilder(TABLE);
msg.append(NEW_LINE);
msg.append(StringUtils.join(cols, TAB));
msg.append(NEW_LINE);
for (List<String> line : lines) {
if (line.size() < cols.size()) {
for (int i = line.size(); i < cols.size(); i++) {
line.add(null);
}
}
msg.append(StringUtils.join(line, TAB));
msg.append(NEW_LINE);
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
}
private InterpreterResult renderGraph(Set<Node> nodes,
Set<Relationship> relationships) {
logger.info("Executing renderGraph method");
List<org.apache.zeppelin.tabledata.Node> nodesList = new ArrayList<>();
List<org.apache.zeppelin.tabledata.Relationship> relsList = new ArrayList<>();
for (Relationship rel : relationships) {
relsList.add(Neo4jConversionUtils.toZeppelinRelationship(rel));
}
Map<String, String> labels = getLabels(true);
for (Node node : nodes) {
nodesList.add(Neo4jConversionUtils.toZeppelinNode(node, labels));
}
return new GraphResult(Code.SUCCESS,
new GraphResult.Graph(nodesList, relsList, labels, getTypes(true), true));
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton()
.createOrGetParallelScheduler(Neo4jCypherInterpreter.class.getName() + this.hashCode(),
Integer.parseInt(getProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY)));
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public void cancel(InterpreterContext context) {
}
}

66
neo4j/src/main/java/org/apache/zeppelin/graph/neo4j/utils/Neo4jConversionUtils.java Normal file
View file

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.graph.neo4j.utils;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.neo4j.driver.v1.types.Node;
import org.neo4j.driver.v1.types.Relationship;
/**
* Neo4jConversionUtils
*/
public class Neo4jConversionUtils {
private Neo4jConversionUtils() {}
private static final String[] LETTERS = "0123456789ABCDEF".split("");
public static final String COLOR_GREY = "#D3D3D3";
public static org.apache.zeppelin.tabledata.Node toZeppelinNode(Node n,
Map<String, String> graphLabels) {
Set<String> labels = new LinkedHashSet<>();
String firstLabel = null;
for (String label : n.labels()) {
if (firstLabel == null) {
firstLabel = label;
}
labels.add(label);
}
return new org.apache.zeppelin.tabledata.Node(n.id(), n.asMap(),
labels);
}
public static org.apache.zeppelin.tabledata.Relationship
toZeppelinRelationship(Relationship r) {
return new org.apache.zeppelin.tabledata.Relationship(r.id(), r.asMap(),
r.startNodeId(), r.endNodeId(), r.type());
}
public static String getRandomLabelColor() {
char[] color = new char[7];
color[0] = '#';
for (int i = 1; i < color.length; i++) {
color[i] = LETTERS[(int) Math.floor(Math.random() * 16)].charAt(0);
}
return new String(color);
}
}

42
neo4j/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,42 @@
[
{
"group": "neo4j",
"name": "neo4j",
"className": "org.apache.zeppelin.graph.neo4j.Neo4jCypherInterpreter",
"properties": {
"neo4j.url": {
"envName": null,
"propertyName": "neo4j.url",
"defaultValue": "bolt://localhost:7687",
"description": "The Neo4j's BOLT url."
},
"neo4j.auth.type": {
"envName": null,
"propertyName": "neo4j.auth.type",
"defaultValue": "BASIC",
"description": "The Neo4j's authentication type (NONE, BASIC)."
},
"neo4j.auth.user": {
"envName": null,
"propertyName": "neo4j.auth.user",
"defaultValue": "",
"description": "The Neo4j user name."
},
"neo4j.auth.password": {
"envName": null,
"propertyName": "neo4j.auth.password",
"defaultValue": "",
"description": "The Neo4j user password."
},
"neo4j.max.concurrency": {
"envName": null,
"propertyName": "neo4j.max.concurrency",
"defaultValue": "50",
"description": "Max concurrency call from Zeppelin to Neo4j server."
}
},
"editor": {
"editOnDblClick": false
}
}
]

249
neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java Normal file
View file

@ -0,0 +1,249 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.graph.neo4j;
import static org.junit.Assert.assertEquals;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.graph.GraphResult;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
import org.neo4j.harness.ServerControls;
import org.neo4j.harness.TestServerBuilders;
import com.google.gson.Gson;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class Neo4jCypherInterpreterTest {
private Neo4jCypherInterpreter interpreter;
private InterpreterContext context;
private static ServerControls server;
private static final Gson gson = new Gson();
private static final String LABEL_PERSON = "Person";
private static final String REL_KNOWS = "KNOWS";
private static final String CYPHER_FOREACH = "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))";
private static final String CHPHER_UNWIND = "UNWIND range(1,1000) as x "
+ "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) "
+ "CREATE (n)-[:%s]->(m)";
@BeforeClass
public static void setUpNeo4jServer() throws Exception {
server = TestServerBuilders.newInProcessBuilder()
.withConfig("dbms.security.auth_enabled","false")
.withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10"))
.withFixture(String.format(CHPHER_UNWIND, REL_KNOWS))
.newServer();
}
@AfterClass
public static void tearDownNeo4jServer() throws Exception {
server.close();
}
@Before
public void setUpZeppelin() {
Properties p = new Properties();
p.setProperty(Neo4jConnectionManager.NEO4J_SERVER_URL, server.boltURI().toString());
p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString());
p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50");
interpreter = new Neo4jCypherInterpreter(p);
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(new InterpreterGroup().getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(null));
}
@After
public void tearDownZeppelin() throws Exception {
interpreter.close();
}
@Test
public void testTableWithArray() {
interpreter.open();
InterpreterResult result = interpreter.interpret("return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResult = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[1,2,3]\n";
assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
result = interpreter.interpret("return 'a' as colA, 'b' as colB, [{key: \"value\"}, {key: 1}] as colC", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResultWithMap = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n";
assertEquals(tableResultWithMap, result.toString().replace("%table ", StringUtils.EMPTY));
}
@Test
public void testCreateIndex() {
interpreter.open();
InterpreterResult result = interpreter.interpret("CREATE INDEX ON :Person(name)", context);
assertEquals(Code.SUCCESS, result.code());
assertEquals(StringUtils.EMPTY, result.toString());
}
@Test
public void testRenderTable() {
interpreter.open();
InterpreterResult result = interpreter.interpret("MATCH (n:Person) "
+ "WHERE n.name IN ['name1', 'name2', 'name3'] "
+ "RETURN n.name AS name, n.age AS age", context);
assertEquals(Code.SUCCESS, result.code());
final String tableResult = "name\tage\n\"name1\"\t1\n\"name2\"\t2\n\"name3\"\t3\n";
assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY));
}
@Test
public void testRenderMap() {
interpreter.open();
final String jsonQuery = "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object";
final String objectKey = "object.key";
final String objectListKey = "object.listKey";
InterpreterResult result = interpreter.interpret(jsonQuery, context);
assertEquals(Code.SUCCESS, result.code());
String[] rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 2);
List<String> header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
assertEquals(header.contains(objectKey), true);
assertEquals(header.contains(objectListKey), true);
List<String> row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
final String query = "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]},"
+ "{key: \"value2\", listKey: [{inner: \"Map12\"}, {inner: \"Map22\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(query, context);
assertEquals(Code.SUCCESS, result.code());
rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB));
assertEquals(header.contains(objectKey), true);
assertEquals(header.contains(objectListKey), true);
row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]");
final String jsonListWithNullQuery = "WITH [{key: \"value\", listKey: null},"
+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(jsonListWithNullQuery, context);
assertEquals(Code.SUCCESS, result.code());
rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(header.contains(objectKey), true);
assertEquals(header.contains(objectListKey), true);
row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY);
assertEquals(row.get(header.indexOf(objectListKey)), "");
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
final String jsonListWithoutListKeyQuery = "WITH [{key: \"value\"},"
+ "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] "
+ "AS array UNWIND array AS object RETURN object";
result = interpreter.interpret(jsonListWithoutListKeyQuery, context);
assertEquals(Code.SUCCESS, result.code());
rows = result.toString().replace("%table ", StringUtils.EMPTY).split(Neo4jCypherInterpreter.NEW_LINE);
assertEquals(rows.length, 3);
header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(header.contains(objectKey), true);
assertEquals(header.contains(objectListKey), true);
row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value");
assertEquals(row.get(header.indexOf(objectListKey)), StringUtils.EMPTY);
row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1));
assertEquals(row.size(), header.size());
assertEquals(row.get(header.indexOf(objectKey)), "value2");
assertEquals(row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]");
}
@Test
public void testRenderNetwork() {
interpreter.open();
InterpreterResult result = interpreter.interpret("MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context);
GraphResult.Graph graph = gson.fromJson(result.toString().replace("%network ", StringUtils.EMPTY), GraphResult.Graph.class);
assertEquals(2, graph.getNodes().size());
assertEquals(true, graph.getNodes().iterator().next().getLabel().equals(LABEL_PERSON));
assertEquals(1, graph.getEdges().size());
assertEquals(true, graph.getEdges().iterator().next().getLabel().equals(REL_KNOWS));
assertEquals(1, graph.getLabels().size());
assertEquals(1, graph.getTypes().size());
assertEquals(true, graph.getLabels().containsKey(LABEL_PERSON));
assertEquals(REL_KNOWS, graph.getTypes().iterator().next());
assertEquals(Code.SUCCESS, result.code());
}
@Test
public void testFallingQuery() {
interpreter.open();
final String ERROR_MSG_EMPTY = "";
InterpreterResult result = interpreter.interpret(StringUtils.EMPTY, context);
assertEquals(Code.SUCCESS, result.code());
assertEquals(ERROR_MSG_EMPTY, result.toString());
result = interpreter.interpret(null, context);
assertEquals(Code.SUCCESS, result.code());
assertEquals(ERROR_MSG_EMPTY, result.toString());
result = interpreter.interpret("MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", context);
assertEquals(Code.ERROR, result.code());
}
}

55
pig/pom.xml
View file

@ -22,9 +22,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -36,6 +37,7 @@
<url>http://zeppelin.apache.org</url>
<properties>
<interpreter.name>pig</interpreter.name>
<pig.version>0.17.0</pig.version>
<hadoop.version>2.6.0</hadoop.version>
<tez.version>0.7.0</tez.version>
@ -175,58 +177,13 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/pig
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/pig
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>

3
pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java
View file

@ -18,7 +18,6 @@
package org.apache.zeppelin.pig;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.pig.PigServer;
import org.apache.pig.impl.logicalLayer.FrontendException;
@ -60,7 +59,7 @@ public class PigInterpreter extends BasePigInterpreter {
}
try {
pigServer = new PigServer(execType);
for (Map.Entry entry : getProperty().entrySet()) {
for (Map.Entry entry : getProperties().entrySet()) {
if (!entry.getKey().toString().startsWith("zeppelin.")) {
pigServer.getPigContext().getProperties().setProperty(entry.getKey().toString(),
entry.getValue().toString());

4
pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
View file

@ -55,7 +55,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
pigServer = getPigInterpreter().getPigServer();
maxResult = Integer.parseInt(getProperty(MAX_RESULTS));
}
@ -159,7 +159,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
return this.pigServer;
}
private PigInterpreter getPigInterpreter() {
private PigInterpreter getPigInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
PigInterpreter pig = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PigInterpreter.class.getName());

3
pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
View file

@ -21,6 +21,7 @@ package org.apache.zeppelin.pig;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.After;
@ -48,7 +49,7 @@ public class PigQueryInterpreterTest {
private InterpreterContext context;
@Before
public void setUp() {
public void setUp() throws InterpreterException {
Properties properties = new Properties();
properties.put("zeppelin.pig.execType", "local");
properties.put("zeppelin.pig.maxResult", "20");

1
pom.xml
View file

@ -76,6 +76,7 @@
<module>bigquery</module>
<module>alluxio</module>
<module>scio</module>
<module>neo4j</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-jupyter</module>

61
python/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,6 +33,7 @@
<name>Zeppelin: Python interpreter</name>
<properties>
<interpreter.name>python</interpreter.name>
<python.py4j.version>0.9.2</python.py4j.version>
<python.test.exclude>
**/PythonInterpreterWithPythonInstalledTest.java,
@ -141,17 +142,6 @@
</executions>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>wagon-maven-plugin</artifactId>
@ -212,47 +202,14 @@
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/python</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/python</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>

4
python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
View file

@ -40,7 +40,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.security.SecureRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -55,7 +55,7 @@ public class IPythonClient {
private final IPythonGrpc.IPythonBlockingStub blockingStub;
private final IPythonGrpc.IPythonStub asyncStub;
private Random random = new Random();
private SecureRandom random = new SecureRandom();
/**
* Construct client for accessing RouteGuide server at {@code host:port}.

67
python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
View file

@ -30,6 +30,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@ -78,6 +79,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
private long ipythonLaunchTimeout;
private String additionalPythonPath;
private String additionalPythonInitFile;
private boolean useBuiltinPy4j = true;
private InterpreterOutputStream interpreterOutput = new InterpreterOutputStream(LOGGER);
@ -92,6 +94,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
* @param additionalPythonPath
*/
public void setAdditionalPythonPath(String additionalPythonPath) {
LOGGER.info("setAdditionalPythonPath: " + additionalPythonPath);
this.additionalPythonPath = additionalPythonPath;
}
@ -105,19 +108,25 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
this.additionalPythonInitFile = additionalPythonInitFile;
}
public void setAddBulitinPy4j(boolean add) {
this.useBuiltinPy4j = add;
}
@Override
public void open() {
public void open() throws InterpreterException {
try {
if (ipythonClient != null) {
// IPythonInterpreter might already been opened by PythonInterpreter
return;
}
pythonExecutable = getProperty().getProperty("zeppelin.python", "python");
pythonExecutable = getProperty("zeppelin.python", "python");
LOGGER.info("Python Exec: " + pythonExecutable);
ipythonLaunchTimeout = Long.parseLong(
getProperty().getProperty("zeppelin.ipython.launch.timeout", "30000"));
getProperty("zeppelin.ipython.launch.timeout", "30000"));
this.zeppelinContext = new PythonZeppelinContext(
getInterpreterGroup().getInterpreterHookRegistry(),
Integer.parseInt(getProperty().getProperty("zeppelin.python.maxResult", "1000")));
Integer.parseInt(getProperty("zeppelin.python.maxResult", "1000")));
int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
LOGGER.info("Launching IPython Kernel at port: " + ipythonPort);
@ -218,29 +227,25 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
watchDog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
executor.setWatchdog(watchDog);
String py4jLibPath = null;
if (System.getenv("ZEPPELIN_HOME") != null) {
py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator
+ PythonInterpreter.ZEPPELIN_PY4JPATH;
} else {
Path workingPath = Paths.get("..").toAbsolutePath();
py4jLibPath = workingPath + File.separator + PythonInterpreter.ZEPPELIN_PY4JPATH;
}
if (additionalPythonPath != null) {
// put the py4j at the end, because additionalPythonPath may already contain py4j.
// e.g. PySparkInterpreter
additionalPythonPath = additionalPythonPath + ":" + py4jLibPath;
} else {
additionalPythonPath = py4jLibPath;
}
Map<String, String> envs = EnvironmentUtils.getProcEnvironment();
if (envs.containsKey("PYTHONPATH")) {
envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH"));
} else {
envs.put("PYTHONPATH", additionalPythonPath);
if (useBuiltinPy4j) {
String py4jLibPath = null;
if (System.getenv("ZEPPELIN_HOME") != null) {
py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator
+ PythonInterpreter.ZEPPELIN_PY4JPATH;
} else {
Path workingPath = Paths.get("..").toAbsolutePath();
py4jLibPath = workingPath + File.separator + PythonInterpreter.ZEPPELIN_PY4JPATH;
}
if (additionalPythonPath != null) {
// put the py4j at the end, because additionalPythonPath may already contain py4j.
// e.g. PySparkInterpreter
additionalPythonPath = additionalPythonPath + ":" + py4jLibPath;
} else {
additionalPythonPath = py4jLibPath;
}
}
LOGGER.debug("PYTHONPATH: " + envs.get("PYTHONPATH"));
Map<String, String> envs = setupIPythonEnv();
executor.execute(cmd, envs, this);
// wait until IPython kernel is started or timeout
@ -272,6 +277,18 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
}
}
protected Map<String, String> setupIPythonEnv() throws IOException {
Map<String, String> envs = EnvironmentUtils.getProcEnvironment();
if (envs.containsKey("PYTHONPATH")) {
if (additionalPythonPath != null) {
envs.put("PYTHONPATH", additionalPythonPath + ":" + envs.get("PYTHONPATH"));
}
} else {
envs.put("PYTHONPATH", additionalPythonPath);
}
return envs;
}
@Override
public void close() {
if (watchDog != null) {

57
python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java
View file

@ -50,6 +50,8 @@ public class PythonCondaInterpreter extends Interpreter {
public static final Pattern PATTERN_COMMAND_HELP = Pattern.compile("help");
public static final Pattern PATTERN_COMMAND_INFO = Pattern.compile("info");
private String currentCondaEnvName = StringUtils.EMPTY;
public PythonCondaInterpreter(Properties property) {
super(property);
}
@ -65,7 +67,8 @@ public class PythonCondaInterpreter extends Interpreter {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
InterpreterOutput out = context.out;
Matcher activateMatcher = PATTERN_COMMAND_ACTIVATE.matcher(st);
Matcher createMatcher = PATTERN_COMMAND_CREATE.matcher(st);
@ -112,8 +115,19 @@ public class PythonCondaInterpreter extends Interpreter {
}
}
public String getCurrentCondaEnvName() {
return currentCondaEnvName;
}
public void setCurrentCondaEnvName(String currentCondaEnvName) {
if (currentCondaEnvName == null) {
currentCondaEnvName = StringUtils.EMPTY;
}
this.currentCondaEnvName = currentCondaEnvName;
}
private void changePythonEnvironment(String envName)
throws IOException, InterruptedException {
throws IOException, InterruptedException, InterpreterException {
PythonInterpreter python = getPythonInterpreter();
String binPath = null;
if (envName == null) {
@ -130,16 +144,17 @@ public class PythonCondaInterpreter extends Interpreter {
}
}
}
setCurrentCondaEnvName(envName);
python.setPythonCommand(binPath);
}
private void restartPythonProcess() {
private void restartPythonProcess() throws InterpreterException {
PythonInterpreter python = getPythonInterpreter();
python.close();
python.open();
}
protected PythonInterpreter getPythonInterpreter() {
protected PythonInterpreter getPythonInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p =
@ -199,7 +214,7 @@ public class PythonCondaInterpreter extends Interpreter {
}
private InterpreterResult runCondaActivate(String envName)
throws IOException, InterruptedException {
throws IOException, InterruptedException, InterpreterException {
if (null == envName || envName.isEmpty()) {
return new InterpreterResult(Code.ERROR, "Env name should be specified");
@ -212,7 +227,7 @@ public class PythonCondaInterpreter extends Interpreter {
}
private InterpreterResult runCondaDeactivate()
throws IOException, InterruptedException {
throws IOException, InterruptedException, InterpreterException {
changePythonEnvironment(null);
restartPythonProcess();
@ -221,8 +236,12 @@ public class PythonCondaInterpreter extends Interpreter {
private String runCondaList() throws IOException, InterruptedException {
List<String> commands = new ArrayList<String>();
commands.add("conda");
commands.add("list");
commands.add(0, "conda");
commands.add(1, "list");
if (!getCurrentCondaEnvName().isEmpty()) {
commands.add(2, "-n");
commands.add(3, getCurrentCondaEnvName());
}
return runCondaCommandForTableOutput("Installed Package List", commands);
}
@ -259,6 +278,10 @@ public class PythonCondaInterpreter extends Interpreter {
restArgs.add(0, "conda");
restArgs.add(1, "install");
restArgs.add(2, "--yes");
if (!getCurrentCondaEnvName().isEmpty()) {
restArgs.add(3, "-n");
restArgs.add(4, getCurrentCondaEnvName());
}
return runCondaCommandForTextOutput("Package Installation", restArgs);
}
@ -269,6 +292,10 @@ public class PythonCondaInterpreter extends Interpreter {
restArgs.add(0, "conda");
restArgs.add(1, "uninstall");
restArgs.add(2, "--yes");
if (!getCurrentCondaEnvName().isEmpty()) {
restArgs.add(3, "-n");
restArgs.add(4, getCurrentCondaEnvName());
}
return runCondaCommandForTextOutput("Package Uninstallation", restArgs);
}
@ -349,10 +376,16 @@ public class PythonCondaInterpreter extends Interpreter {
*/
@Override
public Scheduler getScheduler() {
PythonInterpreter pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
PythonInterpreter pythonInterpreter = null;
try {
pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
return null;
}
} catch (InterpreterException e) {
e.printStackTrace();
return null;
}
}

25
python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java
View file

@ -56,7 +56,8 @@ public class PythonDockerInterpreter extends Interpreter {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
File pythonScript = new File(getPythonInterpreter().getScriptPath());
InterpreterOutput out = context.out;
@ -105,7 +106,7 @@ public class PythonDockerInterpreter extends Interpreter {
}
public void setPythonCommand(String cmd) {
public void setPythonCommand(String cmd) throws InterpreterException {
PythonInterpreter python = getPythonInterpreter();
python.setPythonCommand(cmd);
}
@ -140,21 +141,27 @@ public class PythonDockerInterpreter extends Interpreter {
*/
@Override
public Scheduler getScheduler() {
PythonInterpreter pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
PythonInterpreter pythonInterpreter = null;
try {
pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
return null;
}
} catch (InterpreterException e) {
e.printStackTrace();
return null;
}
}
private void restartPythonProcess() {
private void restartPythonProcess() throws InterpreterException {
PythonInterpreter python = getPythonInterpreter();
python.close();
python.open();
}
protected PythonInterpreter getPythonInterpreter() {
protected PythonInterpreter getPythonInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
@ -173,7 +180,7 @@ public class PythonDockerInterpreter extends Interpreter {
return python;
}
public boolean pull(InterpreterOutput out, String image) {
public boolean pull(InterpreterOutput out, String image) throws InterpreterException {
int exit = 0;
try {
exit = runCommand(out, "docker", "pull", image);

23
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -57,7 +57,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import py4j.GatewayServer;
import py4j.commands.Command;
/**
* Python interpreter for Zeppelin.
@ -101,7 +100,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
File scriptFile = File.createTempFile("zeppelin_python-", ".py", new File("/tmp"));
scriptPath = scriptFile.getAbsolutePath();
} catch (IOException e) {
throw new InterpreterException(e);
throw new RuntimeException(e);
}
}
@ -116,7 +115,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
return path;
}
private void createPythonScript() {
private void createPythonScript() throws InterpreterException {
File out = new File(scriptPath);
if (out.exists() && out.isDirectory()) {
@ -131,7 +130,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
return scriptPath;
}
private void copyFile(File out, String sourceFile) {
private void copyFile(File out, String sourceFile) throws InterpreterException {
ClassLoader classLoader = getClass().getClassLoader();
try {
FileOutputStream outStream = new FileOutputStream(out);
@ -144,7 +143,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
}
}
private void createGatewayServerAndStartScript() throws UnknownHostException {
private void createGatewayServerAndStartScript()
throws UnknownHostException, InterpreterException {
createPythonScript();
if (System.getenv("ZEPPELIN_HOME") != null) {
py4jLibPath = System.getenv("ZEPPELIN_HOME") + File.separator + ZEPPELIN_PY4JPATH;
@ -219,11 +219,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
}
@Override
public void open() {
public void open() throws InterpreterException {
// try IPythonInterpreter first. If it is not available, we will fallback to the original
// python interpreter implementation.
iPythonInterpreter = getIPythonInterpreter();
if (getProperty().getProperty("zeppelin.python.useIPython", "true").equals("true") &&
if (getProperty("zeppelin.python.useIPython", "true").equals("true") &&
iPythonInterpreter.checkIPythonPrerequisite()) {
try {
iPythonInterpreter.open();
@ -369,7 +369,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter)
throws InterpreterException {
if (iPythonInterpreter != null) {
return iPythonInterpreter.interpret(cmd, contextInterpreter);
}
@ -551,7 +552,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
bootstrapCode += line + "\n";
}
interpret(bootstrapCode, context);
try {
interpret(bootstrapCode, context);
} catch (InterpreterException e) {
throw new IOException(e);
}
}
public GUI getGui() {

10
python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java
View file

@ -22,6 +22,7 @@ import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
@ -42,7 +43,7 @@ public class PythonInterpreterPandasSql extends Interpreter {
super(property);
}
PythonInterpreter getPythonInterpreter() {
PythonInterpreter getPythonInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
@ -62,7 +63,7 @@ public class PythonInterpreterPandasSql extends Interpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
LOG.info("Open Python SQL interpreter instance: {}", this.toString());
try {
@ -76,14 +77,15 @@ public class PythonInterpreterPandasSql extends Interpreter {
}
@Override
public void close() {
public void close() throws InterpreterException {
LOG.info("Close Python SQL interpreter instance: {}", this.toString());
Interpreter python = getPythonInterpreter();
python.close();
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
Interpreter python = getPythonInterpreter();

5
python/src/main/resources/grpc/python/ipython_server.py
View file

@ -35,7 +35,7 @@ else:
import queue as queue
TIMEOUT = 30
TIMEOUT = 60*60*24*365*100 # 100 years
class IPython(ipython_pb2_grpc.IPythonServicer):
@ -50,7 +50,8 @@ class IPython(ipython_pb2_grpc.IPythonServicer):
self._status = ipython_pb2.RUNNING
def execute(self, request, context):
print("execute code: " + request.code)
print("execute code:\n")
print(request.code)
sys.stdout.flush()
stdout_queue = queue.Queue(maxsize = 10)
stderr_queue = queue.Queue(maxsize = 10)

13
python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
View file

@ -23,6 +23,7 @@ import org.apache.zeppelin.display.ui.Select;
import org.apache.zeppelin.display.ui.TextBox;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@ -56,7 +57,7 @@ public class IPythonInterpreterTest {
private IPythonInterpreter interpreter;
@Before
public void setUp() {
public void setUp() throws InterpreterException {
Properties properties = new Properties();
interpreter = new IPythonInterpreter(properties);
InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class);
@ -71,11 +72,11 @@ public class IPythonInterpreterTest {
@Test
public void testIPython() throws IOException, InterruptedException {
public void testIPython() throws IOException, InterruptedException, InterpreterException {
testInterpreter(interpreter);
}
public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException {
public static void testInterpreter(final Interpreter interpreter) throws IOException, InterruptedException, InterpreterException {
// to make this test can run under both python2 and python3
InterpreterResult result = interpreter.interpret("from __future__ import print_function", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@ -261,7 +262,11 @@ public class IPythonInterpreterTest {
} catch (InterruptedException e) {
e.printStackTrace();
}
interpreter.cancel(context2);
try {
interpreter.cancel(context2);
} catch (InterpreterException e) {
e.printStackTrace();
}
}
}.start();
result = interpreter.interpret("import time\ntime.sleep(10)", context2);

14
python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java
View file

@ -37,7 +37,7 @@ public class PythonCondaInterpreterTest {
private PythonInterpreter python;
@Before
public void setUp() {
public void setUp() throws InterpreterException {
conda = spy(new PythonCondaInterpreter(new Properties()));
python = mock(PythonInterpreter.class);
@ -57,7 +57,7 @@ public class PythonCondaInterpreterTest {
}
@Test
public void testListEnv() throws IOException, InterruptedException {
public void testListEnv() throws IOException, InterruptedException, InterpreterException {
setMockCondaEnvList();
// list available env
@ -72,23 +72,25 @@ public class PythonCondaInterpreterTest {
}
@Test
public void testActivateEnv() throws IOException, InterruptedException {
public void testActivateEnv() throws IOException, InterruptedException, InterpreterException {
setMockCondaEnvList();
String envname = "env1";
InterpreterContext context = getInterpreterContext();
conda.interpret("activate env1", context);
conda.interpret("activate " + envname, context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(python).setPythonCommand("/path1/bin/python");
assertTrue(envname.equals(conda.getCurrentCondaEnvName()));
}
@Test
public void testDeactivate() {
public void testDeactivate() throws InterpreterException {
InterpreterContext context = getInterpreterContext();
conda.interpret("deactivate", context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(python).setPythonCommand("python");
assertTrue(conda.getCurrentCondaEnvName().isEmpty());
}
@Test

6
python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java
View file

@ -41,7 +41,7 @@ public class PythonDockerInterpreterTest {
private PythonInterpreter python;
@Before
public void setUp() {
public void setUp() throws InterpreterException {
docker = spy(new PythonDockerInterpreter(new Properties()));
python = mock(PythonInterpreter.class);
@ -58,7 +58,7 @@ public class PythonDockerInterpreterTest {
}
@Test
public void testActivateEnv() {
public void testActivateEnv() throws InterpreterException {
InterpreterContext context = getInterpreterContext();
docker.interpret("activate env", context);
verify(python, times(1)).open();
@ -68,7 +68,7 @@ public class PythonDockerInterpreterTest {
}
@Test
public void testDeactivate() {
public void testDeactivate() throws InterpreterException {
InterpreterContext context = getInterpreterContext();
docker.interpret("deactivate", context);
verify(python, times(1)).open();

9
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java
View file

@ -22,6 +22,7 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@ -83,7 +84,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
}
@Test
public void dependenciesAreInstalled() {
public void dependenciesAreInstalled() throws InterpreterException {
// matplotlib
InterpreterResult ret = python.interpret("import matplotlib", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
@ -94,7 +95,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
}
@Test
public void showPlot() throws IOException {
public void showPlot() throws IOException, InterpreterException {
// Simple plot test
InterpreterResult ret;
ret = python.interpret("import matplotlib.pyplot as plt", context);
@ -111,7 +112,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
@Test
// Test for when configuration is set to auto-close figures after show().
public void testClose() throws IOException {
public void testClose() throws IOException, InterpreterException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;
@ -145,7 +146,7 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene
@Test
// Test for when configuration is set to not auto-close figures after show().
public void testNoClose() throws IOException {
public void testNoClose() throws IOException, InterpreterException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;

13
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java
View file

@ -33,6 +33,7 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@ -107,18 +108,18 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
}
@After
public void afterTest() throws IOException {
public void afterTest() throws IOException, InterpreterException {
sql.close();
}
@Test
public void dependenciesAreInstalled() {
public void dependenciesAreInstalled() throws InterpreterException {
InterpreterResult ret = python.interpret("import pandas\nimport pandasql\nimport numpy\n", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
}
@Test
public void errorMessageIfDependenciesNotInstalled() {
public void errorMessageIfDependenciesNotInstalled() throws InterpreterException {
InterpreterResult ret;
ret = sql.interpret("SELECT * from something", context);
@ -128,7 +129,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
}
@Test
public void sqlOverTestDataPrintsTable() throws IOException {
public void sqlOverTestDataPrintsTable() throws IOException, InterpreterException {
InterpreterResult ret;
// given
//String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34";
@ -152,7 +153,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
}
@Test
public void badSqlSyntaxFails() throws IOException {
public void badSqlSyntaxFails() throws IOException, InterpreterException {
//when
InterpreterResult ret = sql.interpret("select wrong syntax", context);
@ -162,7 +163,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener
}
@Test
public void showDataFrame() throws IOException {
public void showDataFrame() throws IOException, InterpreterException {
InterpreterResult ret;
ret = python.interpret("import pandas as pd", context);
ret = python.interpret("import numpy as np", context);

9
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -38,6 +38,7 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@ -64,7 +65,7 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
}
@Before
public void beforeTest() throws IOException {
public void beforeTest() throws IOException, InterpreterException {
cmdHistory = "";
// python interpreter
@ -96,20 +97,20 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
}
@Test
public void testInterpret() throws InterruptedException, IOException {
public void testInterpret() throws InterruptedException, IOException, InterpreterException {
InterpreterResult result = pythonInterpreter.interpret("print (\"hi\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
}
@Test
public void testInterpretInvalidSyntax() throws IOException {
public void testInterpretInvalidSyntax() throws IOException, InterpreterException {
InterpreterResult result = pythonInterpreter.interpret("for x in range(0,3): print (\"hi\")\n", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
}
@Test
public void testRedefinitionZeppelinContext() {
public void testRedefinitionZeppelinContext() throws InterpreterException {
String pyRedefinitionCode = "z = 1\n";
String pyRestoreCode = "z = __zeppelin__\n";
String pyValidCode = "z.input(\"test\")\n";

54
scalding/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,6 +33,7 @@
<name>Zeppelin: Scalding interpreter</name>
<properties>
<interpreter.name>scalding</interpreter.name>
<!--library versions-->
<hadoop.version>2.6.0</hadoop.version>
<scalding.version>0.16.1-RC1</scalding.version>
@ -147,55 +148,14 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/scalding</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/scalding</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<!-- Plugin to compile Scala code -->
<plugin>
<groupId>org.scala-tools</groupId>

8
scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java
View file

@ -69,7 +69,7 @@ public class ScaldingInterpreter extends Interpreter {
@Override
public void open() {
numOpenInstances = numOpenInstances + 1;
String maxOpenInstancesStr = property.getProperty(MAX_OPEN_INSTANCES,
String maxOpenInstancesStr = getProperty(MAX_OPEN_INSTANCES,
MAX_OPEN_INSTANCES_DEFAULT);
int maxOpenInstances = 50;
try {
@ -83,8 +83,8 @@ public class ScaldingInterpreter extends Interpreter {
return;
}
logger.info("Opening instance {}", numOpenInstances);
logger.info("property: {}", property);
String argsString = property.getProperty(ARGS_STRING, ARGS_STRING_DEFAULT);
logger.info("property: {}", getProperties());
String argsString = getProperty(ARGS_STRING, ARGS_STRING_DEFAULT);
String[] args;
if (argsString == null) {
args = new String[0];
@ -121,7 +121,7 @@ public class ScaldingInterpreter extends Interpreter {
return new InterpreterResult(Code.SUCCESS);
}
InterpreterResult interpreterResult = new InterpreterResult(Code.ERROR);
if (property.getProperty(ARGS_STRING).contains("hdfs")) {
if (getProperty(ARGS_STRING).contains("hdfs")) {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());

45
scio/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -34,6 +34,7 @@
<description>Zeppelin Scio support</description>
<properties>
<interpreter.name>scio</interpreter.name>
<!--library versions-->
<scio.version>0.2.4</scio.version>
<guava.version>14.0.1</guava.version> <!-- update needed -->
@ -106,12 +107,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
@ -156,34 +157,6 @@
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/scio</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>

2
scripts/docker/zeppelin/bin/Dockerfile
View file

@ -33,7 +33,7 @@ RUN echo "$LOG_TAG update and install basic packages" && \
apt-get install -y build-essential
RUN echo "$LOG_TAG install tini related packages" && \
apt-get install -y curl grep sed dpkg && \
apt-get install -y wget curl grep sed dpkg && \
TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \
curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
dpkg -i tini.deb && \

57
shell/pom.xml
View file

@ -20,10 +20,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
@ -33,17 +33,18 @@
<name>Zeppelin: Shell interpreter</name>
<properties>
<interpreter.name>sh</interpreter.name>
<!--library versions -->
<commons.lang3.version>3.4</commons.lang3.version>
<commons.exec.version>1.3</commons.exec.version>
<interpreter.name>sh</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@ -79,54 +80,12 @@
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/sh</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/sh</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
</plugins>
</build>

4
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -174,8 +174,8 @@ public class ShellInterpreter extends KerberosInterpreter {
return false;
}
public void createSecureConfiguration() {
Properties properties = getProperty();
public void createSecureConfiguration() throws InterpreterException {
Properties properties = getProperties();
CommandLine cmdLine = CommandLine.parse(shell);
cmdLine.addArgument("-c", false);
String kinitCommand = String.format("kinit -k -t %s %s",

30
spark/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
View file

@ -21,12 +21,15 @@ import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.python.IPythonInterpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
/**
@ -43,18 +46,33 @@ public class IPySparkInterpreter extends IPythonInterpreter {
}
@Override
public void open() {
getProperty().setProperty("zeppelin.python", PySparkInterpreter.getPythonExec(property));
public void open() throws InterpreterException {
setProperty("zeppelin.python",
PySparkInterpreter.getPythonExec(getProperties()));
sparkInterpreter = getSparkInterpreter();
SparkConf conf = sparkInterpreter.getSparkContext().getConf();
String additionalPythonPath = conf.get("spark.submit.pyFiles").replaceAll(",", ":") +
":../interpreter/lib/python";
setAdditionalPythonPath(additionalPythonPath);
// only set PYTHONPATH in local or yarn-client mode.
// yarn-cluster will setup PYTHONPATH automatically.
if (!conf.get("spark.submit.deployMode").equals("cluster")) {
setAdditionalPythonPath(PythonUtils.sparkPythonPath());
setAddBulitinPy4j(false);
}
setAdditionalPythonInitFile("python/zeppelin_ipyspark.py");
super.open();
}
private SparkInterpreter getSparkInterpreter() {
@Override
protected Map<String, String> setupIPythonEnv() throws IOException {
Map<String, String> env = super.setupIPythonEnv();
// set PYSPARK_PYTHON
SparkConf conf = sparkInterpreter.getSparkContext().getConf();
if (conf.contains("spark.pyspark.python")) {
env.put("PYSPARK_PYTHON", conf.get("spark.pyspark.python"));
}
return env;
}
private SparkInterpreter getSparkInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());

61
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -86,11 +86,11 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
File scriptFile = File.createTempFile("zeppelin_pyspark-", ".py");
scriptPath = scriptFile.getAbsolutePath();
} catch (IOException e) {
throw new InterpreterException(e);
throw new RuntimeException(e);
}
}
private void createPythonScript() {
private void createPythonScript() throws InterpreterException {
ClassLoader classLoader = getClass().getClassLoader();
File out = new File(scriptPath);
@ -112,10 +112,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
@Override
public void open() {
public void open() throws InterpreterException {
// try IPySparkInterpreter first
iPySparkInterpreter = getIPySparkInterpreter();
if (property.getProperty("zeppelin.spark.useIPython", "true").equals("true") &&
if (getProperty("zeppelin.pyspark.useIPython", "true").equals("true") &&
iPySparkInterpreter.checkIPythonPrerequisite()) {
try {
iPySparkInterpreter.open();
@ -132,8 +132,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
iPySparkInterpreter = null;
if (property.getProperty("zeppelin.spark.useIPython", "true").equals("true")) {
if (getProperty("zeppelin.pyspark.useIPython", "true").equals("true")) {
// don't print it when it is in testing, just for easy output check in test.
try {
InterpreterContext.get().out.write(("IPython is not available, " +
@ -202,13 +201,18 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
private Map setupPySparkEnv() throws IOException{
private Map setupPySparkEnv() throws IOException, InterpreterException {
Map env = EnvironmentUtils.getProcEnvironment();
if (!env.containsKey("PYTHONPATH")) {
SparkConf conf = getSparkConf();
env.put("PYTHONPATH", conf.get("spark.submit.pyFiles").replaceAll(",", ":") +
":../interpreter/lib/python");
// only set PYTHONPATH in local or yarn-client mode.
// yarn-cluster will setup PYTHONPATH automatically.
SparkConf conf = getSparkConf();
if (!conf.get("spark.submit.deployMode", "client").equals("cluster")) {
if (!env.containsKey("PYTHONPATH")) {
env.put("PYTHONPATH", PythonUtils.sparkPythonPath());
} else {
env.put("PYTHONPATH", PythonUtils.sparkPythonPath());
}
}
// get additional class paths when using SPARK_SUBMIT and not using YARN-CLIENT
@ -223,7 +227,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
LOGGER.debug("PYTHONPATH: " + env.get("PYTHONPATH"));
LOGGER.info("PYTHONPATH: " + env.get("PYTHONPATH"));
// set PYSPARK_PYTHON
if (getSparkConf().contains("spark.pyspark.python")) {
env.put("PYSPARK_PYTHON", getSparkConf().get("spark.pyspark.python"));
}
return env;
}
@ -241,7 +250,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return pythonExec;
}
private void createGatewayServerAndStartScript() {
private void createGatewayServerAndStartScript() throws InterpreterException {
// create python script
createPythonScript();
@ -250,7 +259,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
gatewayServer = new GatewayServer(this, port);
gatewayServer.start();
String pythonExec = getPythonExec(property);
String pythonExec = getPythonExec(getProperties());
LOGGER.info("pythonExec: " + pythonExec);
CommandLine cmd = CommandLine.parse(pythonExec);
cmd.addArgument(scriptPath, false);
cmd.addArgument(Integer.toString(port), false);
@ -289,7 +299,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
private int findRandomOpenPortOnAllLocalInterfaces() {
private int findRandomOpenPortOnAllLocalInterfaces() throws InterpreterException {
int port;
try (ServerSocket socket = new ServerSocket(0);) {
port = socket.getLocalPort();
@ -388,7 +398,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
sparkInterpreter.populateSparkWebUrl(context);
if (sparkInterpreter.isUnsupportedSparkVersion()) {
@ -494,7 +505,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
@Override
public void cancel(InterpreterContext context) {
public void cancel(InterpreterContext context) throws InterpreterException {
if (iPySparkInterpreter != null) {
iPySparkInterpreter.cancel(context);
return;
@ -514,7 +525,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
@Override
public int getProgress(InterpreterContext context) {
public int getProgress(InterpreterContext context) throws InterpreterException {
if (iPySparkInterpreter != null) {
return iPySparkInterpreter.getProgress(context);
}
@ -525,7 +536,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
InterpreterContext interpreterContext) throws InterpreterException {
if (iPySparkInterpreter != null) {
return iPySparkInterpreter.completion(buf, cursor, interpreterContext);
}
@ -626,7 +637,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
private SparkInterpreter getSparkInterpreter() {
private SparkInterpreter getSparkInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
@ -660,7 +671,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return iPySpark;
}
public SparkZeppelinContext getZeppelinContext() {
public SparkZeppelinContext getZeppelinContext() throws InterpreterException {
SparkInterpreter sparkIntp = getSparkInterpreter();
if (sparkIntp != null) {
return getSparkInterpreter().getZeppelinContext();
@ -669,7 +680,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public JavaSparkContext getJavaSparkContext() {
public JavaSparkContext getJavaSparkContext() throws InterpreterException {
SparkInterpreter intp = getSparkInterpreter();
if (intp == null) {
return null;
@ -678,7 +689,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public Object getSparkSession() {
public Object getSparkSession() throws InterpreterException {
SparkInterpreter intp = getSparkInterpreter();
if (intp == null) {
return null;
@ -687,7 +698,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public SparkConf getSparkConf() {
public SparkConf getSparkConf() throws InterpreterException {
JavaSparkContext sc = getJavaSparkContext();
if (sc == null) {
return null;
@ -696,7 +707,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public SQLContext getSQLContext() {
public SQLContext getSQLContext() throws InterpreterException {
SparkInterpreter intp = getSparkInterpreter();
if (intp == null) {
return null;

96
spark/src/main/java/org/apache/zeppelin/spark/PythonUtils.java Normal file
View file

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.List;
/**
* Util class for PySpark
*/
public class PythonUtils {
/**
* Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from ZEPPELIN_HOME
* when it is embedded mode.
*
* This method will called in zeppelin server process and spark driver process when it is
* local or yarn-client mode.
*/
public static String sparkPythonPath() {
List<String> pythonPath = new ArrayList<String>();
String sparkHome = System.getenv("SPARK_HOME");
String zeppelinHome = System.getenv("ZEPPELIN_HOME");
if (zeppelinHome == null) {
zeppelinHome = new File("..").getAbsolutePath();
}
if (sparkHome != null) {
// non-embedded mode when SPARK_HOME is specified.
File pyspark = new File(sparkHome, "python/lib/pyspark.zip");
if (!pyspark.exists()) {
throw new RuntimeException("No pyspark.zip found under " + sparkHome + "/python/lib");
}
pythonPath.add(pyspark.getAbsolutePath());
File[] py4j = new File(sparkHome + "/python/lib").listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith("py4j");
}
});
if (py4j.length == 0) {
throw new RuntimeException("No py4j files found under " + sparkHome + "/python/lib");
} else if (py4j.length > 1) {
throw new RuntimeException("Multiple py4j files found under " + sparkHome + "/python/lib");
} else {
pythonPath.add(py4j[0].getAbsolutePath());
}
} else {
// embedded mode
File pyspark = new File(zeppelinHome, "interpreter/spark/pyspark/pyspark.zip");
if (!pyspark.exists()) {
throw new RuntimeException("No pyspark.zip found: " + pyspark.getAbsolutePath());
}
pythonPath.add(pyspark.getAbsolutePath());
File[] py4j = new File(zeppelinHome, "interpreter/spark/pyspark").listFiles(
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith("py4j");
}
});
if (py4j.length == 0) {
throw new RuntimeException("No py4j files found under " + zeppelinHome +
"/interpreter/spark/pyspark");
} else if (py4j.length > 1) {
throw new RuntimeException("Multiple py4j files found under " + sparkHome +
"/interpreter/spark/pyspark");
} else {
pythonPath.add(py4j[0].getAbsolutePath());
}
}
// add ${ZEPPELIN_HOME}/interpreter/lib/python for all the cases
pythonPath.add(zeppelinHome + "/interpreter/lib/python");
return StringUtils.join(pythonPath, ":");
}
}

117
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -351,7 +351,11 @@ public class SparkInterpreter extends Interpreter {
}
public boolean isYarnMode() {
return getProperty("master").startsWith("yarn");
String master = getProperty("master");
if (master == null) {
master = getProperty("spark.master", "local[*]");
}
return master.startsWith("yarn");
}
/**
@ -371,13 +375,8 @@ public class SparkInterpreter extends Interpreter {
conf.set("spark.executor.uri", execUri);
}
conf.set("spark.scheduler.mode", "FAIR");
conf.setMaster(getProperty("master"));
if (isYarnMode()) {
conf.set("master", "yarn");
conf.set("spark.submit.deployMode", "client");
}
Properties intpProperty = getProperty();
Properties intpProperty = getProperties();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
@ -394,8 +393,6 @@ public class SparkInterpreter extends Interpreter {
}
}
setupConfForPySpark(conf);
setupConfForSparkR(conf);
Class SparkSession = Utils.findClass("org.apache.spark.sql.SparkSession");
Object builder = Utils.invokeStaticMethod(SparkSession, "builder");
Utils.invokeMethod(builder, "config", new Class[]{ SparkConf.class }, new Object[]{ conf });
@ -512,7 +509,7 @@ public class SparkInterpreter extends Interpreter {
}
conf.set("spark.scheduler.mode", "FAIR");
Properties intpProperty = getProperty();
Properties intpProperty = getProperties();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
@ -529,96 +526,10 @@ public class SparkInterpreter extends Interpreter {
}
}
}
setupConfForPySpark(conf);
setupConfForSparkR(conf);
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
}
private void setupConfForPySpark(SparkConf conf) {
Object pysparkBaseProperty =
new DefaultInterpreterProperty("SPARK_HOME", null, null).getValue();
String pysparkBasePath = pysparkBaseProperty != null ? pysparkBaseProperty.toString() : null;
File pysparkPath;
if (null == pysparkBasePath) {
pysparkBasePath =
new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../")
.getValue().toString();
pysparkPath = new File(pysparkBasePath,
"interpreter" + File.separator + "spark" + File.separator + "pyspark");
} else {
pysparkPath = new File(pysparkBasePath,
"python" + File.separator + "lib");
}
//Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
//TODO(zjffdu), this is not maintainable when new version is added.
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip",
"py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip", "py4j-0.10.4-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);
if (libFile.exists()) {
pythonLibUris.add(libFile.toURI().toString());
}
}
pythonLibUris.trimToSize();
// Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
// when spark version is less than or equal to 1.4.1
if (pythonLibUris.size() == 2) {
try {
String confValue = conf.get("spark.yarn.dist.files");
conf.set("spark.yarn.dist.files", confValue + "," + Joiner.on(",").join(pythonLibUris));
} catch (NoSuchElementException e) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
}
if (!useSparkSubmit()) {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
}
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
conf.set("spark.submit.pyFiles", Joiner.on(",").join(pythonLibUris));
}
// Distributes needed libraries to workers
// when spark version is greater than or equal to 1.5.0
if (isYarnMode()) {
conf.set("spark.yarn.isPython", "true");
}
}
private void setupConfForSparkR(SparkConf conf) {
Object sparkRBaseProperty =
new DefaultInterpreterProperty("SPARK_HOME", null, null).getValue();
String sparkRBasePath = sparkRBaseProperty != null ? sparkRBaseProperty.toString() : null;
File sparkRPath;
if (null == sparkRBasePath) {
sparkRBasePath =
new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../")
.getValue().toString();
sparkRPath = new File(sparkRBasePath,
"interpreter" + File.separator + "spark" + File.separator + "R");
} else {
sparkRPath = new File(sparkRBasePath, "R" + File.separator + "lib");
}
sparkRPath = new File(sparkRPath, "sparkr.zip");
if (sparkRPath.exists() && sparkRPath.isFile()) {
String archives = null;
if (conf.contains("spark.yarn.dist.archives")) {
archives = conf.get("spark.yarn.dist.archives");
}
if (archives != null) {
archives = archives + "," + sparkRPath + "#sparkr";
} else {
archives = sparkRPath + "#sparkr";
}
conf.set("spark.yarn.dist.archives", archives);
} else {
logger.warn("sparkr.zip is not found, sparkr may not work.");
}
}
static final String toString(Object o) {
return (o instanceof String) ? (String) o : "";
}
@ -632,19 +543,19 @@ public class SparkInterpreter extends Interpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean(
property.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true"));
getProperty("zeppelin.spark.enableSupportedVersionCheck", "true"));
// set properties and do login before creating any spark stuff for secured cluster
if (isYarnMode()) {
System.setProperty("SPARK_YARN_MODE", "true");
}
if (getProperty().containsKey("spark.yarn.keytab") &&
getProperty().containsKey("spark.yarn.principal")) {
if (getProperties().containsKey("spark.yarn.keytab") &&
getProperties().containsKey("spark.yarn.principal")) {
try {
String keytab = getProperty().getProperty("spark.yarn.keytab");
String principal = getProperty().getProperty("spark.yarn.principal");
String keytab = getProperties().getProperty("spark.yarn.keytab");
String principal = getProperties().getProperty("spark.yarn.principal");
UserGroupInformation.loginUserFromKeytab(principal, keytab);
} catch (IOException e) {
throw new RuntimeException("Can not pass kerberos authentication", e);
@ -1052,7 +963,7 @@ public class SparkInterpreter extends Interpreter {
sparkUrl = getSparkUIUrl();
Map<String, String> infos = new java.util.HashMap<>();
infos.put("url", sparkUrl);
String uiEnabledProp = property.getProperty("spark.ui.enabled", "true");
String uiEnabledProp = getProperty("spark.ui.enabled", "true");
java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean(
uiEnabledProp.trim());
if (!uiEnabled) {

7
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -54,7 +54,7 @@ public class SparkRInterpreter extends Interpreter {
}
@Override
public void open() {
public void open() throws InterpreterException {
String rCmdPath = getProperty("zeppelin.R.cmd");
String sparkRLibPath;
@ -105,7 +105,8 @@ public class SparkRInterpreter extends Interpreter {
}
@Override
public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) {
public InterpreterResult interpret(String lines, InterpreterContext interpreterContext)
throws InterpreterException {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
sparkInterpreter.populateSparkWebUrl(interpreterContext);
@ -220,7 +221,7 @@ public class SparkRInterpreter extends Interpreter {
return new ArrayList<>();
}
private SparkInterpreter getSparkInterpreter() {
private SparkInterpreter getSparkInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());

9
spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
View file

@ -59,7 +59,7 @@ public class SparkSqlInterpreter extends Interpreter {
this.maxResult = Integer.parseInt(getProperty(MAX_RESULTS));
}
private SparkInterpreter getSparkInterpreter() {
private SparkInterpreter getSparkInterpreter() throws InterpreterException {
LazyOpenInterpreter lazy = null;
SparkInterpreter spark = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName());
@ -86,7 +86,8 @@ public class SparkSqlInterpreter extends Interpreter {
public void close() {}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context)
throws InterpreterException {
SQLContext sqlc = null;
SparkInterpreter sparkInterpreter = getSparkInterpreter();
@ -134,7 +135,7 @@ public class SparkSqlInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
public void cancel(InterpreterContext context) throws InterpreterException {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
SQLContext sqlc = sparkInterpreter.getSQLContext();
SparkContext sc = sqlc.sparkContext();
@ -149,7 +150,7 @@ public class SparkSqlInterpreter extends Interpreter {
@Override
public int getProgress(InterpreterContext context) {
public int getProgress(InterpreterContext context) throws InterpreterException {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
return sparkInterpreter.getProgress(context);
}

8
spark/src/main/java/org/apache/zeppelin/spark/SparkZeppelinContext.java
View file

@ -79,7 +79,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext {
}
if (supportedClasses.isEmpty()) {
throw new InterpreterException("Can not load Dataset/DataFrame/SchemaRDD class");
throw new RuntimeException("Can not load Dataset/DataFrame/SchemaRDD class");
}
}
@ -112,7 +112,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext {
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException | ClassCastException e) {
sc.clearJobGroup();
throw new InterpreterException(e);
throw new RuntimeException(e);
}
List<Attribute> columns = null;
@ -129,7 +129,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext {
.asJava();
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
throw new RuntimeException(e);
}
StringBuilder msg = new StringBuilder();
@ -165,7 +165,7 @@ public class SparkZeppelinContext extends BaseZeppelinContext {
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
throw new RuntimeException(e);
}
if (rows.length > maxResult) {

16
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
View file

@ -117,7 +117,7 @@ public class ZeppelinR implements ExecuteResultHandler {
File scriptFile = File.createTempFile("zeppelin_sparkr-", ".R");
scriptPath = scriptFile.getAbsolutePath();
} catch (IOException e) {
throw new InterpreterException(e);
throw new RuntimeException(e);
}
}
@ -125,7 +125,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* Start R repl
* @throws IOException
*/
public void open() throws IOException {
public void open() throws IOException, InterpreterException {
createRScript();
zeppelinR.put(hashCode(), this);
@ -170,7 +170,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* @param expr
* @return
*/
public Object eval(String expr) {
public Object eval(String expr) throws InterpreterException {
synchronized (this) {
rRequestObject = new Request("eval", expr, null);
return request();
@ -182,7 +182,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* @param key
* @param value
*/
public void set(String key, Object value) {
public void set(String key, Object value) throws InterpreterException {
synchronized (this) {
rRequestObject = new Request("set", key, value);
request();
@ -194,7 +194,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* @param key
* @return
*/
public Object get(String key) {
public Object get(String key) throws InterpreterException {
synchronized (this) {
rRequestObject = new Request("get", key, null);
return request();
@ -206,7 +206,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* @param key
* @return
*/
public String getS0(String key) {
public String getS0(String key) throws InterpreterException {
synchronized (this) {
rRequestObject = new Request("getS", key, null);
return (String) request();
@ -217,7 +217,7 @@ public class ZeppelinR implements ExecuteResultHandler {
* Send request to r repl and return response
* @return responseValue
*/
private Object request() throws RuntimeException {
private Object request() throws RuntimeException, InterpreterException {
if (!rScriptRunning) {
throw new RuntimeException("r repl is not running");
}
@ -332,7 +332,7 @@ public class ZeppelinR implements ExecuteResultHandler {
/**
* Create R script in tmp dir
*/
private void createRScript() {
private void createRScript() throws InterpreterException {
ClassLoader classLoader = getClass().getClassLoader();
File out = new File(scriptPath);

4
spark/src/main/resources/interpreter-setting.json
View file

@ -153,9 +153,9 @@
"description": "Python command to run pyspark with",
"type": "string"
},
"zeppelin.spark.useIPython": {
"zeppelin.pyspark.useIPython": {
"envName": null,
"propertyName": "zeppelin.spark.useIPython",
"propertyName": "zeppelin.pyspark.useIPython",
"defaultValue": true,
"description": "whether use IPython when it is available",
"type": "checkbox"

6
spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
View file

@ -24,6 +24,7 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
@ -55,10 +56,11 @@ public class IPySparkInterpreterTest {
private InterpreterGroup intpGroup;
@Before
public void setup() {
public void setup() throws InterpreterException {
Properties p = new Properties();
p.setProperty("spark.master", "local[4]");
p.setProperty("master", "local[4]");
p.setProperty("spark.submit.deployMode", "client");
p.setProperty("spark.app.name", "Zeppelin Test");
p.setProperty("zeppelin.spark.useHiveContext", "true");
p.setProperty("zeppelin.spark.maxResult", "1000");
@ -89,7 +91,7 @@ public class IPySparkInterpreterTest {
}
@Test
public void testBasics() throws InterruptedException, IOException {
public void testBasics() throws InterruptedException, IOException, InterpreterException {
// all the ipython test should pass too.
IPythonInterpreterTest.testInterpreter(iPySparkInterpreter);

14
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterMatplotlibTest.java
View file

@ -64,7 +64,7 @@ public class PySparkInterpreterMatplotlibTest {
* normally handles this in real use cases.
*/
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
public InterpreterResult interpret(String st, InterpreterContext context) throws InterpreterException {
context.out.clear();
InterpreterResult result = super.interpret(st, context);
List<InterpreterResultMessage> resultMessages = null;
@ -89,7 +89,7 @@ public class PySparkInterpreterMatplotlibTest {
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.pyspark.python", "python");
p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
p.setProperty("zeppelin.spark.useIPython", "false");
p.setProperty("zeppelin.pyspark.useIPython", "false");
return p;
}
@ -140,7 +140,7 @@ public class PySparkInterpreterMatplotlibTest {
}
@Test
public void dependenciesAreInstalled() {
public void dependenciesAreInstalled() throws InterpreterException {
// matplotlib
InterpreterResult ret = pyspark.interpret("import matplotlib", context);
assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code());
@ -151,7 +151,7 @@ public class PySparkInterpreterMatplotlibTest {
}
@Test
public void showPlot() {
public void showPlot() throws InterpreterException {
// Simple plot test
InterpreterResult ret;
ret = pyspark.interpret("import matplotlib.pyplot as plt", context);
@ -168,7 +168,7 @@ public class PySparkInterpreterMatplotlibTest {
@Test
// Test for when configuration is set to auto-close figures after show().
public void testClose() {
public void testClose() throws InterpreterException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;
@ -195,7 +195,7 @@ public class PySparkInterpreterMatplotlibTest {
@Test
// Test for when configuration is set to not auto-close figures after show().
public void testNoClose() {
public void testNoClose() throws InterpreterException {
InterpreterResult ret;
InterpreterResult ret1;
InterpreterResult ret2;
@ -222,7 +222,7 @@ public class PySparkInterpreterMatplotlibTest {
@Test
// Test angular mode
public void testAngular() {
public void testAngular() throws InterpreterException {
InterpreterResult ret;
ret = pyspark.interpret("import matplotlib.pyplot as plt", context);
ret = pyspark.interpret("plt.close()", context);

Some files were not shown because too many files have changed in this diff Show more