Merge remote-tracking branch 'origin/master' into livyInterperter

This commit is contained in:
Prabhjyot Singh 2016-04-04 12:52:14 +05:30
commit 557d1e13ed
20 changed files with 237 additions and 60 deletions

1
.gitignore vendored
View file

@ -12,6 +12,7 @@
spark/derby.log
spark/metastore_db
spark-1.*-bin-hadoop*
.spark-dist
zeppelin-server/derby.log
lens/lens-cli-hist.log

View file

@ -16,6 +16,9 @@
language: java
sudo: false
cache:
directories:
- .spark-dist
matrix:
include:
@ -48,6 +51,7 @@ matrix:
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "ls -la .spark-dist"
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
@ -58,6 +62,7 @@ before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
- tail conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS

View file

@ -35,7 +35,7 @@
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<flink.version>0.10.0</flink.version>
<flink.version>1.0.0</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<flink.scala.binary.version>2.10</flink.scala.binary.version>
<flink.scala.version>2.10.4</flink.scala.version>
@ -73,25 +73,25 @@
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<artifactId>flink-clients_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<artifactId>flink-runtime_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala</artifactId>
<artifactId>flink-scala_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala-shell</artifactId>
<artifactId>flink-scala-shell_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>

11
pom.xml
View file

@ -242,6 +242,7 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
@ -250,6 +251,7 @@
<target>1.7</target>
</configuration>
</plugin>
<!-- Test coverage plugin -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
@ -271,6 +273,7 @@
</execution>
</executions>
</plugin>
<!-- Checkstyle plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -489,6 +492,7 @@
<exclude>conf/notebook-authorization.json</exclude>
<exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
<exclude>.spark-dist/**</exclude>
<!-- bundled from bootstrap -->
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
@ -641,6 +645,13 @@
</lifecycleMappingMetadata>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
</plugin>
</plugins>
</pluginManagement>
</build>

View file

@ -23,3 +23,4 @@
- nodejs
- maven
- python-addons
- r

View file

@ -0,0 +1,24 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# defaults variables for r role
---
r_cran_mirror: http://cran.rstudio.com/
r_repository:
- type: deb
url: "{{ r_cran_mirror }}/bin/linux/ubuntu {{ ansible_distribution_release }}/"
r_packages_repos: "{{ r_cran_mirror }}"

View file

@ -0,0 +1,65 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Install R binaries and CRAN packages
---
# The Ubuntu archives on CRAN are signed with a key with ID E084DAB9. Add this key to the system.
- name: repository | add public key
apt_key:
id: E084DAB9
keyserver: keyserver.ubuntu.com
state: present
# in order to get the latest version of R, add CRAN repository to the to the list of sources
- name: repository | add cran-r
apt_repository:
repo: "{{item.type}} {{item.url}}"
state: present
update_cache: true
with_items: r_repository
- name: Install R. This may take a while.
apt: pkg=r-base state=present
- name: openssl and libcurl required for R devtools package
apt: pkg={{item}} state=present
with_items:
- libssl-dev
- libcurl4-openssl-dev
# knitr will also pull in the `evaluate` R package as a dependency
- name: Install R packages required for the R interpreter. This may take a while.
shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))"
with_items:
- knitr
- devtools
- name: Install rCharts (requires devtools first).
shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('rCharts' %in% installed.packages()[,'Package'])) devtools::install_github('rCharts', 'ramnathv')"
- name: Install R repr package recommended for the R interpreter display system (requires devtools first).
shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('repr' %in% installed.packages()[,'Package'])) devtools::install_github('IRkernel/repr')"
- name: Install R packages recommended for the R interpreter.
shell: /usr/bin/Rscript --slave --no-save --no-restore-history -e "if (! ('{{item}}' %in% installed.packages()[,'Package'])) install.packages(pkgs=c('{{item}}'), repos=c('{{r_packages_repos}}'))"
with_items:
- ggplot2
- googleVis
- mplot
- htmltools
- base64enc
- data.table

View file

@ -32,9 +32,9 @@ echo
echo 'cd /vagrant/incubator-zeppelin'
echo 'mvn clean package -DskipTests'
echo
echo '# or for a specific build'
echo '# or for a specific Spark/Hadoop build with additional options such as python and R support'
echo
echo 'mvn clean package -Pspark-1.5 -Ppyspark -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests'
echo 'mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests'
echo './bin/zeppelin-daemon.sh start'
echo
echo 'On your host machine browse to http://localhost:8080/'

View file

@ -50,7 +50,11 @@
<akka.group>org.spark-project.akka</akka.group>
<akka.version>2.3.4-spark</akka.version>
<spark.download.url>http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz</spark.download.url>
<spark.archive>spark-${spark.version}</spark.archive>
<spark.download.url>
http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz
</spark.download.url>
<spark.dist.cache>${project.build.directory}/../../.spark-dist</spark.dist.cache>
<py4j.version>0.8.2.1</py4j.version>
</properties>
@ -787,12 +791,12 @@
</goals>
<configuration>
<url>${spark.download.url}</url>
<unpack>true</unpack>
<outputDirectory>${project.build.directory}/spark-dist</outputDirectory>
<outputDirectory>${spark.dist.cache}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
@ -806,13 +810,28 @@
</filesets>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<id>download-and-zip-pyspark-files</id>
<id>unzip-pyspark-files</id>
<phase>validate</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<untar src="${spark.dist.cache}/${spark.archive}.tgz"
dest="${project.build.directory}/spark-dist"
compression="gzip"/>
</target>
</configuration>
</execution>
<execution>
<id>zip-pyspark-files</id>
<phase>generate-resources</phase>
<goals>
<goal>run</goal>
@ -821,9 +840,9 @@
<target>
<delete dir="../interpreter/spark/pyspark"/>
<copy todir="../interpreter/spark/pyspark"
file="${project.build.directory}/spark-dist/spark-${spark.version}/python/lib/py4j-${py4j.version}-src.zip"/>
file="${project.build.directory}/spark-dist/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/>
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
basedir="${project.build.directory}/spark-dist/spark-${spark.version}/python"
basedir="${project.build.directory}/spark-dist/${spark.archive}/python"
includes="pyspark/*.py,pyspark/**/*.py"/>
</target>
</configuration>

View file

@ -17,7 +17,7 @@
#
if [ $# -ne 2 ]; then
if [[ "$#" -ne 2 ]]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
@ -26,10 +26,10 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
if [[ "$?" -eq 0 ]]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
if [[ "$?" -eq 0 ]]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
@ -40,31 +40,52 @@ fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
TIMEOUT_SEC=590
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
SPARK_CACHE=".spark-dist"
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
if [ ! -d "${SPARK_HOME}" ]; then
if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
STARTTIME=`date +%s`
timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
if [[ ! -d "${SPARK_HOME}" ]]; then
mkdir -p "${SPARK_CACHE}"
cd "${SPARK_CACHE}"
if [[ ! -f "${SPARK_ARCHIVE}.tgz" ]]; then
pwd
ls -la .
echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..."
# download archive if not cached
if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
STARTTIME=`date +%s`
timeout -s KILL "${TIMEOUT_SEC}" wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
ENDTIME=`date +%s`
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz?asjson=1")
STARTTIME=`date +%s`
timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}"
ENDTIME=`date +%s`
DOWNLOADTIME=$((ENDTIME-STARTTIME))
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
STARTTIME=`date +%s`
timeout -s KILL "${TIMEOUT_SEC}" wget -q "${PREFFERED}${PATHINFO}"
ENDTIME=`date +%s`
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
fi
fi
# extract archive in un-cached root, clean-up on failure
cp "${SPARK_ARCHIVE}.tgz" ..
cd ..
if ! tar zxf "${SPARK_ARCHIVE}.tgz" ; then
echo "Unable to extract ${SPARK_ARCHIVE}.tgz" >&2
rm -rf "${SPARK_ARCHIVE}"
rm -f "${SPARK_ARCHIVE}.tgz"
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
set +xe

View file

@ -17,7 +17,7 @@
#
if [ $# -ne 2 ]; then
if [[ "$#" -ne 2 ]]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
@ -26,10 +26,10 @@ fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
if [[ "$?" -eq 0 ]]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
if [[ "$?" -eq 0 ]]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
@ -38,17 +38,18 @@ else
SPARK_VER_RANGE=">1.3"
fi
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
# create PID dir. test case detect pid file so they can select active spark home dir for test
mkdir -p ${SPARK_HOME}/run
export SPARK_PID_DIR=${SPARK_HOME}/run
export SPARK_PID_DIR="${SPARK_HOME}/run"
mkdir -p "${SPARK_PID_DIR}"
# start
export SPARK_MASTER_PORT=7071

View file

@ -16,7 +16,7 @@
# limitations under the License.
#
if [ $# -ne 2 ]; then
if [[ "$#" -ne 2 ]]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
@ -27,12 +27,15 @@ HADOOP_VERSION="${2}"
set -xe
FWDIR=$(dirname "${BASH_SOURCE-$0}")
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
echo "SPARK_HOME is ${SPARK_HOME}"
# set create PID dir
export SPARK_PID_DIR=${SPARK_HOME}/run
export SPARK_PID_DIR="${SPARK_HOME}/run"
${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
${SPARK_HOME}/sbin/stop-master.sh

View file

@ -369,8 +369,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.6</version>
<executions>
<execution>
<id>start-zeppelin</id>

View file

@ -47,10 +47,10 @@
<webXml>dist\WEB-INF\web.xml</webXml>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>

View file

@ -81,8 +81,7 @@
ngToastProvider.configure({
dismissButton: true,
dismissOnClick: false,
timeout: 6000,
verticalPosition: 'bottom'
timeout: 6000
});
});

View file

@ -75,7 +75,7 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
// add missing field of option
if (!setting.option) {
setting.option = {};
}
}
if (setting.option.remote === undefined) {
// remote always true for now
setting.option.remote = true;
@ -94,7 +94,7 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
}).
error(function (data, status, headers, config) {
console.log('Error %o %o', status, data.message);
ngToast.danger(data.message);
ngToast.danger({content: data.message, verticalPosition: 'bottom'});
form.$show();
});
}
@ -210,7 +210,7 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
}).
error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
ngToast.danger(data.message);
ngToast.danger({content: data.message, verticalPosition: 'bottom'});
});
};

View file

@ -16,7 +16,7 @@
angular.module('zeppelinWebApp')
.controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location,
$timeout, $compile, websocketMsgSrv) {
$timeout, $compile, websocketMsgSrv, ngToast) {
var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_';
$scope.parentNote = null;
$scope.paragraph = null;
@ -39,8 +39,13 @@ angular.module('zeppelinWebApp')
websocketMsgSrv.runParagraph(paragraph.id, paragraph.title, paragraph.text,
paragraph.config, paragraph.settings.params);
} else {
// Error message here
ngToast.danger({content: 'Cannot find a paragraph with id \'' + paragraphId + '\'',
verticalPosition: 'top', dismissOnTimeout: false});
}
} else {
ngToast.danger({
content: 'Please provide a \'paragraphId\' when calling z.runParagraph(paragraphId)',
verticalPosition: 'top', dismissOnTimeout: false});
}
},
@ -49,6 +54,11 @@ angular.module('zeppelinWebApp')
// Only push to server if there paragraphId is defined
if (paragraphId) {
websocketMsgSrv.clientBindAngularObject($routeParams.noteId, varName, value, paragraphId);
} else {
ngToast.danger({
content: 'Please provide a \'paragraphId\' when calling ' +
'z.angularBind(varName, value, \'PUT_HERE_PARAGRAPH_ID\')',
verticalPosition: 'top', dismissOnTimeout: false});
}
},
@ -57,6 +67,11 @@ angular.module('zeppelinWebApp')
// Only push to server if paragraphId is defined
if (paragraphId) {
websocketMsgSrv.clientUnbindAngularObject($routeParams.noteId, varName, paragraphId);
} else {
ngToast.danger({
content: 'Please provide a \'paragraphId\' when calling ' +
'z.angularUnbind(varName, \'PUT_HERE_PARAGRAPH_ID\')',
verticalPosition: 'top', dismissOnTimeout: false});
}
}
};

View file

@ -33,3 +33,7 @@ body {
border-color: white;
}
.ng-toast.ng-toast--top {
top: 100px;
}

View file

@ -73,3 +73,7 @@ body {
.noteAction button.btn {
border-radius: 4px !important;
}
.ng-toast.ng-toast--top {
top: 100px;
}

View file

@ -89,3 +89,7 @@ body {
.nv-controlsWrap {
display: block;
}
.ng-toast.ng-toast--top {
top: 100px;
}