Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Change condition to make pyspark, py4j libraries be distributed to yarn executors

Browse source
This commit is contained in:
Mina Lee 2016-02-21 20:24:27 +09:00
parent 39417c073b
commit 6465ba8902
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -326,7 +326,10 @@ public class SparkInterpreter extends Interpreter {
}
}
pythonLibUris.trimToSize();
if (pythonLibs.length == pythonLibUris.size()) {
// Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
// when spark version is less than or equal to 1.4.1
if (pythonLibUris.size() == 2) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
if (!useSparkSubmit()) {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
@ -334,7 +337,8 @@ public class SparkInterpreter extends Interpreter {
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
}
// Distributes needed libraries to workers.
// Distributes needed libraries to workers
// when spark version is greater than or equal to 1.5.0
if (getProperty("master").equals("yarn-client")) {
conf.set("spark.yarn.isPython", "true");
}