diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index ba7f017236..b5603c8ed4 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -81,8 +81,11 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
# This will evantually passes SPARK_APP_JAR to classpath of SparkIMain
ZEPPELIN_CLASSPATH=${SPARK_APP_JAR}
+ pattern="$SPARK_HOME/python/lib/py4j-*-src.zip"
+ py4j=($pattern)
+ # pick the first match py4j zip - there should only be one
export PYTHONPATH="$SPARK_HOME/python/:$PYTHONPATH"
- export PYTHONPATH="$SPARK_HOME/python/lib/py4j-0.8.2.1-src.zip:$PYTHONPATH"
+ export PYTHONPATH="${py4j[0]}:$PYTHONPATH"
else
# add Hadoop jars into classpath
if [[ -n "${HADOOP_HOME}" ]]; then
@@ -95,7 +98,11 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
fi
addJarInDir "${INTERPRETER_DIR}/dep"
- PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip"
+
+ pattern="${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-*-src.zip"
+ py4j=($pattern)
+ # pick the first match py4j zip - there should only be one
+ PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${py4j[0]}"
if [[ -z "${PYTHONPATH}" ]]; then
export PYTHONPATH="${PYSPARKPATH}"
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index 3af0678a9c..58665b6420 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -33,7 +33,7 @@
Zeppelin: Spark dependencies
Zeppelin spark support
http://zeppelin.incubator.apache.org
-
+
1.4.1
@@ -130,117 +130,117 @@
-
- org.apache.hadoop
- hadoop-yarn-api
- ${yarn.version}
-
-
- asm
- asm
-
-
- org.ow2.asm
- asm
-
-
- org.jboss.netty
- netty
-
-
- commons-logging
- commons-logging
-
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-common
- ${yarn.version}
-
-
- asm
- asm
-
-
- org.ow2.asm
- asm
-
-
- org.jboss.netty
- netty
-
-
- javax.servlet
- servlet-api
-
-
- commons-logging
- commons-logging
-
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-server-web-proxy
- ${yarn.version}
-
-
- asm
- asm
-
-
- org.ow2.asm
- asm
-
-
- org.jboss.netty
- netty
-
-
- javax.servlet
- servlet-api
-
-
- commons-logging
- commons-logging
-
-
-
-
-
- org.apache.hadoop
- hadoop-yarn-client
- ${yarn.version}
-
-
- asm
- asm
-
-
- org.ow2.asm
- asm
-
-
- org.jboss.netty
- netty
-
-
- javax.servlet
- servlet-api
-
-
- commons-logging
- commons-logging
-
-
+
+ org.apache.hadoop
+ hadoop-yarn-api
+ ${yarn.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ org.jboss.netty
+ netty
+
+
+ commons-logging
+ commons-logging
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+ ${yarn.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ org.jboss.netty
+ netty
+
+
+ javax.servlet
+ servlet-api
+
+
+ commons-logging
+ commons-logging
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-server-web-proxy
+ ${yarn.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ org.jboss.netty
+ netty
+
+
+ javax.servlet
+ servlet-api
+
+
+ commons-logging
+ commons-logging
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-client
+ ${yarn.version}
+
+
+ asm
+ asm
+
+
+ org.ow2.asm
+ asm
+
+
+ org.jboss.netty
+ netty
+
+
+ javax.servlet
+ servlet-api
+
+
+ commons-logging
+ commons-logging
+
+
-
-
+
+
org.apache.spark
@@ -489,7 +489,7 @@
-
+
cassandra-spark-1.5
@@ -513,7 +513,27 @@
-
+
+
+ spark-1.6
+
+ 1.5.2
+
+ com.typesafe.akka
+ 2.3.11
+ 2.5.0
+
+
+
+
+
+
+
hadoop-0.23
+
+
+
+
org.slf4j
@@ -72,7 +129,7 @@
guava
14.0.1
-
+
org.apache.maven
@@ -294,18 +351,6 @@
-
-
- vendor-repo
-
-
- cloudera
- https://repository.cloudera.com/artifactory/cloudera-repos/
-
-
-
-
-
@@ -397,7 +442,7 @@
false
false
true
- runtime
+ runtime
${project.groupId}
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 095970f67c..2f2829ea30 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -317,7 +317,8 @@ public class SparkInterpreter extends Interpreter {
"python" + File.separator + "lib");
}
- String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
+ //Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
+ String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip"};
ArrayList pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index a362938860..eb1c0a2d20 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -32,9 +32,10 @@ public class SparkVersion {
public static final SparkVersion SPARK_1_4_0 = SparkVersion.fromVersionString("1.4.0");
public static final SparkVersion SPARK_1_5_0 = SparkVersion.fromVersionString("1.5.0");
public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0");
+ public static final SparkVersion SPARK_1_7_0 = SparkVersion.fromVersionString("1.7.0");
public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0;
- public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_6_0;
+ public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_7_0;
private int version;
private String versionString;