Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-18] Running pyspark without deploying python libraries to every yarn node

Browse source 
- Fixed checkstyle
This commit is contained in:
Jongyoul Lee 2015-06-25 01:33:14 +09:00
parent 929333dcc4
commit 94bdf30e55
1 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -269,20 +269,20 @@ public class SparkInterpreter extends Interpreter {
conf.set(key, val);
}
}
//TODO(jongyoul): Move these codes into PySparkInterpreter.java
String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
String[] pythonLibs = new String[] {"pyspark.zip", "py4j-0.8.2.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib: pythonLibs) {
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(zeppelinPythonLibPath, lib);
if(libFile.exists()) {
if (libFile.exists()) {
pythonLibUris.add(libFile.toURI().toString());
}
}
pythonLibUris.trimToSize();
if(pythonLibs.length == pythonLibUris.size()) {
if (pythonLibs.length == pythonLibUris.size()) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));