Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

handle spark.files correctly for pyspark when spark-submit is used

Browse source
This commit is contained in:
Lee moon soo 2015-09-03 18:55:04 -07:00
parent d4acd1beb8
commit a8a3440eb2
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -280,7 +280,6 @@ public class SparkInterpreter extends Interpreter {
}
//TODO(jongyoul): Move these codes into PySparkInterpreter.java
String pysparkBasePath = getSystemDefault("SPARK_HOME", "spark.home", null);
File pysparkPath;
if (null == pysparkBasePath) {
@ -303,10 +302,13 @@ public class SparkInterpreter extends Interpreter {
pythonLibUris.trimToSize();
if (pythonLibs.length == pythonLibUris.size()) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
if (!useSparkSubmit()) {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
}
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
}
SparkContext sparkContext = new SparkContext(conf);
return sparkContext;
}
@ -315,6 +317,10 @@ public class SparkInterpreter extends Interpreter {
return (o instanceof String) ? (String) o : "";
}
private boolean useSparkSubmit() {
return null != System.getenv("SPARK_HOME");
}
public static String getSystemDefault(
String envName,
String propertyName,