Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ZEPPELIN-2195. Use PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON over zeppelin.pyspark.python

Browse source
This commit is contained in:
Jeff Zhang 2017-02-28 13:57:53 +08:00
parent 897a6398d1
commit fd89a1e30c
2 changed files with 19 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
docs/interpreter/spark.md
View file

@ -104,9 +104,16 @@ You can also set other Spark properties which are not listed in the table. For a
<td>Local repository for dependency loader</td>
</tr>
<tr>
<td>zeppelin.pyspark.python</td>
<td>PYSPARK_PYTHON</td>
<td>python</td>
<td>Python command to run pyspark with</td>
<td>Python binary executable to use for PySpark in both driver and workers (default is <code>python</code> if available, otherwise <code>python</code>).
Property <code>spark.pyspark.python</code> take precedence if it is set</td>
</tr>
<tr>
<td>PYSPARK_DRIVER_PYTHON</td>
<td>python</td>
<td>Python binary executable to use for PySpark in driver only (default is <code>PYSPARK_PYTHON</code>).
Property <code>spark.pyspark.driver.python</code> take precedence if it is set</td>
</tr>
<tr>
<td>zeppelin.spark.concurrentSQL</td>

11
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -200,7 +200,16 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
gatewayServer.start();
// Run python shell
CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python"));
// Choose python in the order of
// PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON > zeppelin.pyspark.python
String pythonExec = getProperty("zeppelin.pyspark.python");
if (System.getenv("PYSPARK_PYTHON") != null) {
pythonExec = System.getenv("PYSPARK_PYTHON");
}
if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
pythonExec = System.getenv("PYSPARK_DRIVER_PYTHON");
}
CommandLine cmd = CommandLine.parse(pythonExec);
cmd.addArgument(scriptPath, false);
cmd.addArgument(Integer.toString(port), false);
cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);