diff --git a/pom.xml b/pom.xml index 7a2ee86f2c..e5f7d9ac4a 100755 --- a/pom.xml +++ b/pom.xml @@ -241,6 +241,7 @@ org.apache.rat apache-rat-plugin + maven-compiler-plugin 3.1 @@ -249,6 +250,7 @@ 1.7 + org.codehaus.mojo @@ -270,6 +272,7 @@ + org.apache.maven.plugins @@ -641,6 +644,13 @@ + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 5cf080201b..02d24e2df5 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -50,7 +50,11 @@ org.spark-project.akka 2.3.4-spark - http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz + spark-${spark.version} + + http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz + + ${project.build.directory}/../../.spark-dist 0.8.2.1 @@ -787,12 +791,12 @@ ${spark.download.url} - true - ${project.build.directory}/../../.spark-dist + ${spark.dist.cache} + maven-clean-plugin @@ -801,18 +805,33 @@ ${basedir}/../python/build - ${project.build.directory}/../../.spark-dist + ${project.build.directory}/spark-dist + org.apache.maven.plugins maven-antrun-plugin - 1.7 - download-and-zip-pyspark-files + unzip-pyspark-files + validate + + run + + + + + + + + + + zip-pyspark-files generate-resources run @@ -821,9 +840,9 @@ + file="${project.build.directory}/spark-dist/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/> diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh index 3619960166..30a3404f25 100755 --- a/testing/downloadSpark.sh +++ b/testing/downloadSpark.sh @@ -45,13 +45,14 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" SPARK_CACHE=".spark-dist" SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" -export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}" +export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" echo "SPARK_HOME is ${SPARK_HOME}" if [[ ! -d "${SPARK_HOME}" ]]; then mkdir -p "${SPARK_CACHE}" cd "${SPARK_CACHE}" if [[ ! -f "${SPARK_ARCHIVE}.tgz" ]]; then + echo "Cache does not have ${SPARK_ARCHIVE} downloading ..." # download archive if not cached if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then # spark 1.1.x and spark 1.2.x can be downloaded from archive @@ -74,7 +75,9 @@ if [[ ! -d "${SPARK_HOME}" ]]; then fi fi - # extract archive, clean-up on failure + # extract archive in un-cached root, clean-up on failure + cp "${SPARK_ARCHIVE}.tgz" .. + cd .. if ! tar zxf "${SPARK_ARCHIVE}.tgz" ; then echo "Unable to extract ${SPARK_ARCHIVE}.tgz" >&2 rm -rf "${SPARK_ARCHIVE}" diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh index 4b1f9d149b..dc7613de30 100755 --- a/testing/startSparkCluster.sh +++ b/testing/startSparkCluster.sh @@ -43,9 +43,8 @@ set -xe FWDIR="$(dirname "${BASH_SOURCE-$0}")" ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" -SPARK_CACHE=".spark-dist" SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" -export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}" +export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" echo "SPARK_HOME is ${SPARK_HOME}" # create PID dir. test case detect pid file so they can select active spark home dir for test diff --git a/testing/stopSparkCluster.sh b/testing/stopSparkCluster.sh index d563e705fc..e049ec42a8 100755 --- a/testing/stopSparkCluster.sh +++ b/testing/stopSparkCluster.sh @@ -30,13 +30,12 @@ set -xe FWDIR="$(dirname "${BASH_SOURCE-$0}")" ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" -SPARK_CACHE=".spark-dist" SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" -export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_CACHE}/${SPARK_ARCHIVE}" +export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" echo "SPARK_HOME is ${SPARK_HOME}" # set create PID dir -export SPARK_PID_DIR=${SPARK_HOME}/run +export SPARK_PID_DIR="${SPARK_HOME}/run" ${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1 ${SPARK_HOME}/sbin/stop-master.sh diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index ee03c334d7..f2c9ced7d6 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -369,8 +369,8 @@ + org.apache.maven.plugins maven-antrun-plugin - 1.6 start-zeppelin diff --git a/zeppelin-web/pom.xml b/zeppelin-web/pom.xml index 21f17dfda7..8878e9a8cd 100644 --- a/zeppelin-web/pom.xml +++ b/zeppelin-web/pom.xml @@ -47,10 +47,10 @@ dist\WEB-INF\web.xml + org.apache.rat apache-rat-plugin - 0.11 **/.idea/