Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

- Removed setting SPARK_HOME

Browse source 
- Changed the location of pyspark's directory into interpreter/spark
This commit is contained in:
Jongyoul Lee 2015-07-03 12:28:54 +09:00
parent ef240f5c39
commit c9cda291c2
3 changed files with 34 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
bin/interpreter.sh
View file

@ -73,16 +73,20 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
$(mkdir -p "${ZEPPELIN_LOG_DIR}")
fi
if [[ x"" == x${PYTHONPATH} ]]; then
export PYTHONPATH="${ZEPPELIN_HOME}/python/lib/pyspark.zip:${ZEPPELIN_HOME}/python/lib/py4j-0.8.2.1-src.zip"
if [[ ! -z "${SPARK_HOME}" ]]; then
PYSPARKPATH="${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip"
else
export PYTHONPATH="$PYTHONPATH:${ZEPPELIN_HOME}/lib/pyspark.zip:${ZEPPELIN_HOME}/python/lib/py4j-0.8.2.1-src.zip"
PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip"
fi
if [[ x"" == x${SPARK_HOME} ]]; then
export SPARK_HOME=${ZEPPELIN_HOME}
if [[ x"" == x"${PYTHONPATH}" ]]; then
export PYTHONPATH="${PYSPARKPATH}"
else
export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
fi
unset PYSPARKPATH
${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
pid=$!

15
spark/pom.xml
View file

@ -779,13 +779,14 @@
</goals>
<configuration>
<target>
<delete dir="../python"/>
<copy todir="../python">
<fileset dir="${project.build.directory}/spark-dist/spark-${spark.version}/python"/>
</copy>
<unzip src="../python/lib/py4j-0.8.2.1-src.zip"
dest="../python/build"/>
<zip destfile="${project.build.directory}/../../python/lib/pyspark.zip"
<delete dir="../interpreter/spark/pyspark"/>
<copy todir="../interpreter/spark/pyspark"
file="${project.build.directory}/spark-dist/spark-${spark.version}/python/lib/py4j-0.8.2.1-src.zip"/>
<!--<fileset dir="${project.build.directory}/spark-dist/spark-${spark.version}/python"/>
</copy>-->
<!--<unzip src="../interpreter/spark/python/py4j-0.8.2.1-src.zip"
dest="../interpreter/spark/python/build"/>-->
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
basedir="${project.build.directory}/spark-dist/spark-${spark.version}/python"
includes="pyspark/*.py,pyspark/**/*.py"/>
</target>

21
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -271,14 +271,27 @@ public class SparkInterpreter extends Interpreter {
}
//TODO(jongyoul): Move these codes into PySparkInterpreter.java
String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
// String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
// File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
String pysparkBasePath = getSystemDefault("SPARK_HOME", "spark.home", null);
File pysparkPath;
if (null == pysparkBasePath) {
pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
pysparkPath = new File(pysparkBasePath,
"interpreter" + File.separator + "spark" + File.separator + "pyspark");
} else {
pysparkPath = new File(pysparkBasePath,
"python" + File.separator + "lib");
}
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(zeppelinPythonLibPath, lib);
// File libFile = new File(zeppelinPythonLibPath, lib);
File libFile = new File(pysparkPath, lib);
if (libFile.exists()) {
pythonLibUris.add(libFile.toURI().toString());
pythonLibUris.add(libFile.toURI().toString());
}
}
pythonLibUris.trimToSize();