Merge remote-tracking branch 'origin/master' into livyInterperter

# Conflicts:
#	testing/startSparkCluster.sh

.gitignore

@@ -45,6 +45,7 @@ zeppelin-web/bower_components
 **/data/
 **/build/
 **/testing/
+!/testing/
 
 # OS generated files #
 ######################

.travis.yml

@@ -53,6 +53,7 @@ install:
   - mvn $BUILD_FLAG $PROFILE -B
 
 before_script:
+  - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
   - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
   - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
 

testing/downloadSpark.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+if [ $# -ne 2 ]; then
+    echo "usage) $0 [spark version] [hadoop version]"
+    echo "   eg) $0 1.3.1 2.6"
+    exit 1
+fi
+
+SPARK_VERSION="${1}"
+HADOOP_VERSION="${2}"
+
+echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
+if [ $? -eq 0 ]; then
+  echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
+  if [ $? -eq 0 ]; then
+    SPARK_VER_RANGE="<=1.2"
+  else
+    SPARK_VER_RANGE="<=1.3"
+  fi
+else
+  SPARK_VER_RANGE=">1.3"
+fi
+
+set -xe
+
+FWDIR=$(dirname "${BASH_SOURCE-$0}")
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
+echo "SPARK_HOME is ${SPARK_HOME}"
+if [ ! -d "${SPARK_HOME}" ]; then
+    if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
+        # spark 1.1.x and spark 1.2.x can be downloaded from archive
+        STARTTIME=`date +%s`
+        timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
+        ENDTIME=`date +%s`
+        DOWNLOADTIME=$((ENDTIME-STARTTIME))
+    else
+        # spark 1.3.x and later can be downloaded from mirror
+        # get download address from mirror
+        MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
+
+        PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
+        PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
+
+        STARTTIME=`date +%s`
+        timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}"
+        ENDTIME=`date +%s`
+        DOWNLOADTIME=$((ENDTIME-STARTTIME))
+    fi
+    tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
+fi
+
+set +xe

testing/startSparkCluster.sh

@@ -38,27 +38,13 @@ else
   SPARK_VER_RANGE=">1.3"
 fi
 
+
 set -xe
 
 FWDIR=$(dirname "${BASH_SOURCE-$0}")
 ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
 export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
 echo "SPARK_HOME is ${SPARK_HOME}"
-if [ ! -d "${SPARK_HOME}" ]; then
-    if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
-        # spark 1.1.x and spark 1.2.x can be downloaded from archive
-        wget -q http://archive.apache.org/dist/spark/spark-$1.4.1/spark-1.4.1-bin-hadoop2.3.tgz
-    else
-        # spark 1.3.x and later can be downloaded from mirror
-        # get download address from mirror
-        MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
-
-        PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
-        PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
-        wget -q "${PREFFERED}${PATHINFO}"
-    fi
-    tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-fi
 
 # create PID dir. test case detect pid file so they can select active spark home dir for test
 mkdir -p ${SPARK_HOME}/run

zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java

@@ -19,6 +19,8 @@ package org.apache.zeppelin;
 
 
 import com.google.common.base.Function;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
 import org.openqa.selenium.*;
 import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.support.ui.FluentWait;
@@ -170,6 +172,7 @@ abstract public class AbstractZeppelinIT {
   protected void handleException(String message, Exception e) throws Exception {
     LOG.error(message, e);
     File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
+    LOG.error("ScreenShot::\ndata:image/png;base64," + new String(Base64.encodeBase64(FileUtils.readFileToByteArray(scrFile))));
     throw e;
   }
 

zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java

@@ -59,7 +59,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
     driver.quit();
   }
   @Test
-  public void testCreateNewButton() throws InterruptedException {
+  public void testCreateNewButton() throws Exception {
     if (!endToEndTestEnabled()) {
       return;
     }
@@ -127,18 +127,15 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
       ZeppelinITUtils.sleep(1000, false);
       deleteTestNotebook(driver);
 
-    } catch (ElementNotVisibleException e) {
-            LOG.error("Exception in ParagraphActionsIT while testCreateNewButton ", e);
-            File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
-            throw e;
-
+    } catch (Exception e) {
+       handleException("Exception in ParagraphActionsIT while testCreateNewButton ", e);
     }
 
 
   }
 
   @Test
-  public void testRemoveButton() throws InterruptedException {
+  public void testRemoveButton() throws Exception {
     if (!endToEndTestEnabled()) {
       return;
     }
@@ -166,9 +163,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
       deleteTestNotebook(driver);
 
     } catch (Exception e) {
-        LOG.error("Exception in ParagraphActionsIT while testRemoveButton ", e);
-        File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
-        throw e;
+      handleException("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
     }
   }
   

zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java

@@ -121,7 +121,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
 
       // check expected text
       waitForText("myVar=1", By.xpath(
-              getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
+              getParagraphXPath(3) + "//div[contains(@id,\"_text\") and @class=\"text\"]"));
 
       /*
        * Click element
@@ -162,7 +162,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
 
       // check expected text by watcher
       waitForText("myVar=3", By.xpath(
-              getParagraphXPath(3) + "//div[@ng-bind=\"paragraph.result.msg\"]"));
+              getParagraphXPath(3) + "//div[contains(@id,\"_text\") and @class=\"text\"]"));
 
       /*
        * Unbind
@@ -196,7 +196,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
           "//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
       sleep(100, true);
 
-      System.out.println("testCreateNotebook Test executed");
+      LOG.info("testCreateNotebook Test executed");
     } catch (Exception e) {
       handleException("Exception in ZeppelinIT while testAngularDisplay ", e);
     }

zeppelin-server/src/test/scala/org/apache/zeppelin/AbstractFunctionalSuite.scala

@@ -52,6 +52,8 @@ class AbstractFunctionalSuite extends FunSuite with WebBrowser with BeforeAndAft
 
   override def afterAll() = {
     "../bin/zeppelin-daemon.sh stop" !
+
+    webDriver.close()
   }
 
   def getDriver(): WebDriver = {