Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ZEPPELIN-1918. Fix build with Spark 2.1.0

Browse source
This commit is contained in:
Jeff Zhang 2017-01-10 15:09:49 +08:00
parent e94d5c0fb6
commit def502fb70
4 changed files with 64 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.travis.yml
View file

@ -40,14 +40,14 @@ matrix:
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.1.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with spark 2.0.2 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with spark 2.1.0 and scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pbeam -Phelium-dev -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""

16
spark-dependencies/pom.xml
View file

@ -523,9 +523,6 @@
<profile>
<id>spark-2.0</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.0.2</spark.version>
<protobuf.version>2.5.0</protobuf.version>
@ -534,6 +531,19 @@
</properties>
</profile>
<profile>
<id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.1.0</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.4</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>
<profile>
<id>hadoop-0.23</id>
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a

16
spark/pom.xml
View file

@ -515,9 +515,6 @@
<profile>
<id>spark-2.0</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.0.2</spark.version>
<protobuf.version>2.5.0</protobuf.version>
@ -526,6 +523,19 @@
</properties>
</profile>
<profile>
<id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.1.0</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.4</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>
<profile>
<id>hadoop-0.23</id>
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a

37
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -1452,8 +1452,10 @@ public class SparkInterpreter extends Interpreter {
.getConstructor(new Class[]{
SparkConf.class, File.class, SecurityManager.class, int.class, String.class});
return constructor.newInstance(new Object[] {
conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"});
Object securityManager = createSecurityManager(conf);
return constructor.newInstance(new Object[]{
conf, outputDir, securityManager, 0, "HTTP Server"});
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e) {
// fallback to old constructor
@ -1464,7 +1466,7 @@ public class SparkInterpreter extends Interpreter {
.getConstructor(new Class[]{
File.class, SecurityManager.class, int.class, String.class});
return constructor.newInstance(new Object[] {
outputDir, new SecurityManager(conf), 0, "HTTP Server"});
outputDir, createSecurityManager(conf), 0, "HTTP Server"});
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e1) {
logger.error(e1.getMessage(), e1);
@ -1472,4 +1474,33 @@ public class SparkInterpreter extends Interpreter {
}
}
}
/**
* Constructor signature of SecurityManager changes in spark 2.10, so we use this method to create
* Security properly for different versions of spark
*
* @param conf
* @return
* @throws ClassNotFoundException
* @throws NoSuchMethodException
* @throws IllegalAccessException
* @throws InvocationTargetException
* @throws InstantiationException
*/
private Object createSecurityManager(SparkConf conf) throws ClassNotFoundException,
NoSuchMethodException, IllegalAccessException, InvocationTargetException,
InstantiationException {
Object securityManager = null;
try {
Constructor<?> smConstructor = getClass().getClassLoader()
.loadClass("org.apache.spark.SecurityManager")
.getConstructor(new Class[]{ SparkConf.class });
} catch (NoSuchMethodException e) {
Constructor<?> smConstructor = getClass().getClassLoader()
.loadClass("org.apache.spark.SecurityManager")
.getConstructor(new Class[]{ SparkConf.class, scala.Option.class });
securityManager = smConstructor.newInstance(conf, null);
}
return securityManager;
}
}