diff --git a/pom.xml b/pom.xml
index 7a2ee86f2c..e5f7d9ac4a 100755
--- a/pom.xml
+++ b/pom.xml
@@ -241,6 +241,7 @@
org.apache.rat
apache-rat-plugin
+
maven-compiler-plugin
3.1
@@ -249,6 +250,7 @@
1.7
+
org.codehaus.mojo
@@ -270,6 +272,7 @@
+
org.apache.maven.plugins
@@ -641,6 +644,13 @@
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+ 1.7
+
+
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index 5cf080201b..02d24e2df5 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -50,7 +50,11 @@
org.spark-project.akka
2.3.4-spark
- http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz
+ spark-${spark.version}
+
+ http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz
+
+ ${project.build.directory}/../../.spark-dist
0.8.2.1
@@ -787,12 +791,12 @@
${spark.download.url}
- true
- ${project.build.directory}/../../.spark-dist
+ ${spark.dist.cache}
+
maven-clean-plugin
@@ -801,18 +805,33 @@
${basedir}/../python/build
- ${project.build.directory}/../../.spark-dist
+ ${project.build.directory}/spark-dist
+
org.apache.maven.plugins
maven-antrun-plugin
- 1.7
- download-and-zip-pyspark-files
+ unzip-pyspark-files
+ validate
+
+ run
+
+
+
+
+
+
+
+
+
+ zip-pyspark-files
generate-resources
run
@@ -821,9 +840,9 @@
+ file="${project.build.directory}/spark-dist/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/>
diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh
index 3619960166..30a3404f25 100755
--- a/testing/downloadSpark.sh
+++ b/testing/downloadSpark.sh
@@ -45,13 +45,14 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
SPARK_CACHE=".spark-dist"
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}"
+export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
if [[ ! -d "${SPARK_HOME}" ]]; then
mkdir -p "${SPARK_CACHE}"
cd "${SPARK_CACHE}"
if [[ ! -f "${SPARK_ARCHIVE}.tgz" ]]; then
+ echo "Cache does not have ${SPARK_ARCHIVE} downloading ..."
# download archive if not cached
if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
@@ -74,7 +75,9 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
fi
fi
- # extract archive, clean-up on failure
+ # extract archive in un-cached root, clean-up on failure
+ cp "${SPARK_ARCHIVE}.tgz" ..
+ cd ..
if ! tar zxf "${SPARK_ARCHIVE}.tgz" ; then
echo "Unable to extract ${SPARK_ARCHIVE}.tgz" >&2
rm -rf "${SPARK_ARCHIVE}"
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
index 4b1f9d149b..dc7613de30 100755
--- a/testing/startSparkCluster.sh
+++ b/testing/startSparkCluster.sh
@@ -43,9 +43,8 @@ set -xe
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-SPARK_CACHE=".spark-dist"
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}"
+export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
# create PID dir. test case detect pid file so they can select active spark home dir for test
diff --git a/testing/stopSparkCluster.sh b/testing/stopSparkCluster.sh
index d563e705fc..e049ec42a8 100755
--- a/testing/stopSparkCluster.sh
+++ b/testing/stopSparkCluster.sh
@@ -30,13 +30,12 @@ set -xe
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-SPARK_CACHE=".spark-dist"
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}"
+export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
# set create PID dir
-export SPARK_PID_DIR=${SPARK_HOME}/run
+export SPARK_PID_DIR="${SPARK_HOME}/run"
${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
${SPARK_HOME}/sbin/stop-master.sh
diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml
index ee03c334d7..f2c9ced7d6 100644
--- a/zeppelin-server/pom.xml
+++ b/zeppelin-server/pom.xml
@@ -369,8 +369,8 @@
+ org.apache.maven.plugins
maven-antrun-plugin
- 1.6
start-zeppelin
diff --git a/zeppelin-web/pom.xml b/zeppelin-web/pom.xml
index 21f17dfda7..8878e9a8cd 100644
--- a/zeppelin-web/pom.xml
+++ b/zeppelin-web/pom.xml
@@ -47,10 +47,10 @@
dist\WEB-INF\web.xml
+
org.apache.rat
apache-rat-plugin
- 0.11
**/.idea/