mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge with master
This commit is contained in:
Eric Charles
commit
d1f0521f5b
19 changed files with 262 additions and 77 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -12,6 +12,7 @@
|
|||
spark/derby.log
|
||||
spark/metastore_db
|
||||
spark-1.*-bin-hadoop*
|
||||
.spark-dist
|
||||
zeppelin-server/derby.log
|
||||
|
||||
lens/lens-cli-hist.log
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@
|
|||
language: java
|
||||
|
||||
sudo: false
|
||||
cache:
|
||||
directories:
|
||||
- .spark-dist
|
||||
|
||||
addons:
|
||||
apt:
|
||||
|
|
@ -55,6 +58,7 @@ matrix:
|
|||
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
|
||||
before_install:
|
||||
- "ls -la .spark-dist"
|
||||
- "export DISPLAY=:99.0"
|
||||
- "sh -e /etc/init.d/xvfb start"
|
||||
|
||||
|
|
@ -69,7 +73,8 @@ install:
|
|||
before_script:
|
||||
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
|
||||
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
|
||||
- echo -e "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER\nexport ZEPPELIN_R_CMD=Rscript" > conf/zeppelin-env.sh
|
||||
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
|
||||
- tail conf/zeppelin-env.sh
|
||||
|
||||
script:
|
||||
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
|
||||
|
|
|
|||
|
|
@ -132,6 +132,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
|
|||
|
||||
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
|
||||
fi
|
||||
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
|
||||
if [[ -n "${HBASE_CONF_DIR}" ]]; then
|
||||
ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}"
|
||||
elif [[ -n "${HBASE_HOME}" ]]; then
|
||||
ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf"
|
||||
else
|
||||
echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded"
|
||||
fi
|
||||
fi
|
||||
|
||||
addJarInDir "${LOCAL_INTERPRETER_REPO}"
|
||||
|
|
|
|||
|
|
@ -64,3 +64,9 @@
|
|||
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
|
||||
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
|
||||
|
||||
#### HBase interpreter configuration ####
|
||||
|
||||
## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
|
||||
|
||||
# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be
|
||||
# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml
|
||||
|
|
|
|||
|
|
@ -10,7 +10,18 @@ group: manual
|
|||
[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed.
|
||||
To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart)
|
||||
|
||||
> Note: currently only HBase 1.0.x releases are supported.
|
||||
## HBase release supported
|
||||
By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1.1.x releases, use the following build command:
|
||||
```bash
|
||||
# HBase 1.1.4
|
||||
mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0
|
||||
```
|
||||
To work with HBase 1.2.0+, use the following build command:
|
||||
|
||||
```bash
|
||||
# HBase 1.2.0
|
||||
mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.2.0 -Dhbase.hadoop.version=2.6.0
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
|
|
@ -23,7 +34,7 @@ To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/
|
|||
<tr>
|
||||
<td>hbase.home</td>
|
||||
<td>/usr/lib/hbase</td>
|
||||
<td>Installation directory of Hbase</td>
|
||||
<td>Installation directory of HBase, defaults to HBASE_HOME in environment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>hbase.ruby.sources</td>
|
||||
|
|
@ -31,12 +42,31 @@ To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/
|
|||
<td>Path to Ruby scripts relative to 'hbase.home'</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>hbase.test.mode</td>
|
||||
<td>zeppelin.hbase.test.mode</td>
|
||||
<td>false</td>
|
||||
<td>Disable checks for unit and manual tests</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
If you want to connect to HBase running on a cluster, you'll need to follow the next step.
|
||||
|
||||
### Export HBASE_HOME
|
||||
In **conf/zeppelin-env.sh**, export `HBASE_HOME` environment variable with your HBase installation path. This ensures `hbase-site.xml` can be loaded.
|
||||
|
||||
for example
|
||||
|
||||
```bash
|
||||
export HBASE_HOME=/usr/lib/hbase
|
||||
```
|
||||
|
||||
or, when running with CDH
|
||||
|
||||
```bash
|
||||
export HBASE_HOME="/opt/cloudera/parcels/CDH/lib/hbase"
|
||||
```
|
||||
|
||||
You can optionally export `HBASE_CONF_DIR` instead of `HBASE_HOME` should you have custom HBase configurations.
|
||||
|
||||
## Enabling the HBase Shell Interpreter
|
||||
|
||||
In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**.
|
||||
|
|
|
|||
|
|
@ -249,12 +249,34 @@ public class FlinkInterpreter extends Interpreter {
|
|||
Code r = null;
|
||||
|
||||
String incomplete = "";
|
||||
boolean inComment = false;
|
||||
|
||||
for (int l = 0; l < linesToRun.length; l++) {
|
||||
final String s = linesToRun[l];
|
||||
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
|
||||
if (l + 1 < linesToRun.length) {
|
||||
String nextLine = linesToRun[l + 1].trim();
|
||||
if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
|
||||
boolean continuation = false;
|
||||
if (nextLine.isEmpty()
|
||||
|| nextLine.startsWith("//") // skip empty line or comment
|
||||
|| nextLine.startsWith("}")
|
||||
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
|
||||
continuation = true;
|
||||
} else if (!inComment && nextLine.startsWith("/*")) {
|
||||
inComment = true;
|
||||
continuation = true;
|
||||
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
|
||||
inComment = false;
|
||||
continuation = true;
|
||||
} else if (nextLine.length() > 1
|
||||
&& nextLine.charAt(0) == '.'
|
||||
&& nextLine.charAt(1) != '.' // ".."
|
||||
&& nextLine.charAt(1) != '/') { // "./"
|
||||
continuation = true;
|
||||
} else if (inComment) {
|
||||
continuation = true;
|
||||
}
|
||||
if (continuation) {
|
||||
incomplete += s + "\n";
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,22 @@ public class FlinkInterpreterTest {
|
|||
flink.destroy();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextLineInvocation() {
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n.toInt", context).code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextLineComments() {
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextLineCompanionObject() {
|
||||
String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret(code, context).code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleStatement() {
|
||||
InterpreterResult result = flink.interpret("val a=1", context);
|
||||
|
|
@ -63,12 +79,6 @@ public class FlinkInterpreterTest {
|
|||
assertEquals("1", result.message());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextlineInvoke() {
|
||||
InterpreterResult result = flink.interpret("\"123\"\n .toInt", context);
|
||||
assertEquals("res0: Int = 123\n", result.message());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWordCount() {
|
||||
flink.interpret("val text = env.fromElements(\"To be or not to be\")", context);
|
||||
|
|
|
|||
|
|
@ -37,21 +37,20 @@ import java.util.List;
|
|||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* Support for Hbase Shell. All the commands documented here
|
||||
* Support for HBase Shell. All the commands documented here
|
||||
* http://hbase.apache.org/book.html#shell is supported.
|
||||
*
|
||||
* Requirements:
|
||||
* Hbase Shell should be installed on the same machine. To be more specific, the following dir.
|
||||
* HBase Shell should be installed on the same machine. To be more specific, the following dir.
|
||||
* should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
|
||||
* Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure
|
||||
* HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure
|
||||
* that the client is configured properly.
|
||||
*
|
||||
* The interpreter takes 3 config parameters:
|
||||
* hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/
|
||||
* hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/
|
||||
* hbase.ruby.sources: Dir where shell ruby code is installed.
|
||||
* Path is relative to hbase.home. Default: lib/ruby
|
||||
* hbase.irb.load: (Testing only) Default is true.
|
||||
* Whether to load irb in the interpreter.
|
||||
* zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
|
||||
*/
|
||||
public class HbaseInterpreter extends Interpreter {
|
||||
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
|
||||
|
|
@ -62,11 +61,13 @@ public class HbaseInterpreter extends Interpreter {
|
|||
static {
|
||||
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
|
||||
new InterpreterPropertyBuilder()
|
||||
.add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
|
||||
.add("hbase.home",
|
||||
getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
|
||||
"Installation directory of HBase")
|
||||
.add("hbase.ruby.sources", "lib/ruby",
|
||||
"Path to Ruby scripts relative to 'hbase.home'")
|
||||
.add("hbase.test.mode", "false", "Disable checks for unit and manual tests")
|
||||
.build());
|
||||
.add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
|
||||
.build());
|
||||
}
|
||||
|
||||
public HbaseInterpreter(Properties property) {
|
||||
|
|
@ -79,7 +80,7 @@ public class HbaseInterpreter extends Interpreter {
|
|||
this.writer = new StringWriter();
|
||||
scriptingContainer.setOutput(this.writer);
|
||||
|
||||
if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) {
|
||||
if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
|
||||
String hbase_home = getProperty("hbase.home");
|
||||
String ruby_src = getProperty("hbase.ruby.sources");
|
||||
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
|
||||
|
|
@ -89,7 +90,7 @@ public class HbaseInterpreter extends Interpreter {
|
|||
|
||||
File f = abs_ruby_src.toFile();
|
||||
if (!f.exists() || !f.isDirectory()) {
|
||||
throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src
|
||||
throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src
|
||||
+ "'");
|
||||
}
|
||||
|
||||
|
|
@ -155,4 +156,24 @@ public class HbaseInterpreter extends Interpreter {
|
|||
return null;
|
||||
}
|
||||
|
||||
private static String getSystemDefault(
|
||||
String envName,
|
||||
String propertyName,
|
||||
String defaultValue) {
|
||||
|
||||
if (envName != null && !envName.isEmpty()) {
|
||||
String envValue = System.getenv().get(envName);
|
||||
if (envValue != null) {
|
||||
return envValue;
|
||||
}
|
||||
}
|
||||
|
||||
if (propertyName != null && !propertyName.isEmpty()) {
|
||||
String propValue = System.getProperty(propertyName);
|
||||
if (propValue != null) {
|
||||
return propValue;
|
||||
}
|
||||
}
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ public class HbaseInterpreterTest {
|
|||
Properties properties = new Properties();
|
||||
properties.put("hbase.home", "");
|
||||
properties.put("hbase.ruby.sources", "");
|
||||
properties.put("hbase.test.mode", "true");
|
||||
properties.put("zeppelin.hbase.test.mode", "true");
|
||||
|
||||
hbaseInterpreter = new HbaseInterpreter(properties);
|
||||
hbaseInterpreter.open();
|
||||
|
|
@ -72,4 +72,4 @@ public class HbaseInterpreterTest {
|
|||
assertEquals(InterpreterResult.Code.ERROR, result.code());
|
||||
assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
11
pom.xml
11
pom.xml
|
|
@ -241,6 +241,7 @@
|
|||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
|
|
@ -249,6 +250,7 @@
|
|||
<target>1.7</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- Test coverage plugin -->
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
|
|
@ -270,6 +272,7 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- Checkstyle plugin -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
|
|
@ -488,6 +491,7 @@
|
|||
<exclude>conf/notebook-authorization.json</exclude>
|
||||
<exclude>conf/zeppelin-env.sh</exclude>
|
||||
<exclude>spark-*-bin*/**</exclude>
|
||||
<exclude>.spark-dist/**</exclude>
|
||||
|
||||
<!-- bundled from bootstrap -->
|
||||
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
|
||||
|
|
@ -640,6 +644,13 @@
|
|||
</lifecycleMappingMetadata>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.7</version>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
|
|
|
|||
|
|
@ -209,13 +209,34 @@ public class ScaldingInterpreter extends Interpreter {
|
|||
out.reset();
|
||||
Code r = null;
|
||||
String incomplete = "";
|
||||
boolean inComment = false;
|
||||
|
||||
for (int l = 0; l < linesToRun.length; l++) {
|
||||
String s = linesToRun[l];
|
||||
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
|
||||
if (l + 1 < linesToRun.length) {
|
||||
String nextLine = linesToRun[l + 1].trim();
|
||||
if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
|
||||
boolean continuation = false;
|
||||
if (nextLine.isEmpty()
|
||||
|| nextLine.startsWith("//") // skip empty line or comment
|
||||
|| nextLine.startsWith("}")
|
||||
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
|
||||
continuation = true;
|
||||
} else if (!inComment && nextLine.startsWith("/*")) {
|
||||
inComment = true;
|
||||
continuation = true;
|
||||
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
|
||||
inComment = false;
|
||||
continuation = true;
|
||||
} else if (nextLine.length() > 1
|
||||
&& nextLine.charAt(0) == '.'
|
||||
&& nextLine.charAt(1) != '.' // ".."
|
||||
&& nextLine.charAt(1) != '/') { // "./"
|
||||
continuation = true;
|
||||
} else if (inComment) {
|
||||
continuation = true;
|
||||
}
|
||||
if (continuation) {
|
||||
incomplete += s + "\n";
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,6 +88,17 @@ public class ScaldingInterpreterTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextLineComments() {
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNextLineCompanionObject() {
|
||||
String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicIntp() {
|
||||
assertEquals(InterpreterResult.Code.SUCCESS,
|
||||
|
|
|
|||
|
|
@ -50,8 +50,11 @@
|
|||
<akka.group>org.spark-project.akka</akka.group>
|
||||
<akka.version>2.3.4-spark</akka.version>
|
||||
|
||||
<spark.download.url>http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz</spark.download.url>
|
||||
<spark.bin.download.url>http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}-bin-without-hadoop.tgz</spark.bin.download.url>
|
||||
<spark.archive>spark-${spark.version}</spark.archive>
|
||||
<spark.download.url>
|
||||
http://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz
|
||||
</spark.download.url>
|
||||
<spark.dist.cache>${project.build.directory}/../../.spark-dist</spark.dist.cache>
|
||||
<py4j.version>0.8.2.1</py4j.version>
|
||||
</properties>
|
||||
|
||||
|
|
@ -788,12 +791,12 @@
|
|||
</goals>
|
||||
<configuration>
|
||||
<url>${spark.download.url}</url>
|
||||
<unpack>true</unpack>
|
||||
<outputDirectory>${project.build.directory}/spark-dist</outputDirectory>
|
||||
<outputDirectory>${spark.dist.cache}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<configuration>
|
||||
|
|
@ -807,13 +810,28 @@
|
|||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-and-zip-pyspark-files</id>
|
||||
<id>unzip-pyspark-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<untar src="${spark.dist.cache}/${spark.archive}.tgz"
|
||||
dest="${project.build.directory}/spark-dist"
|
||||
compression="gzip"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
|
||||
<execution>
|
||||
<id>zip-pyspark-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
|
|
@ -822,11 +840,11 @@
|
|||
<target>
|
||||
<delete dir="../interpreter/spark/pyspark"/>
|
||||
<copy todir="../interpreter/spark/pyspark"
|
||||
file="${project.build.directory}/spark-dist/spark-${spark.version}/python/lib/py4j-${py4j.version}-src.zip"/>
|
||||
file="${project.build.directory}/spark-dist/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/>
|
||||
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
|
||||
basedir="${project.build.directory}/spark-dist/spark-${spark.version}/python"
|
||||
basedir="${project.build.directory}/spark-dist/${spark.archive}/python"
|
||||
includes="pyspark/*.py,pyspark/**/*.py"/>
|
||||
</target>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
#
|
||||
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
if [[ "$#" -ne 2 ]]; then
|
||||
echo "usage) $0 [spark version] [hadoop version]"
|
||||
echo " eg) $0 1.3.1 2.6"
|
||||
exit 1
|
||||
|
|
@ -26,10 +26,10 @@ fi
|
|||
SPARK_VERSION="${1}"
|
||||
HADOOP_VERSION="${2}"
|
||||
|
||||
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
SPARK_VER_RANGE="<=1.2"
|
||||
else
|
||||
SPARK_VER_RANGE="<=1.3"
|
||||
|
|
@ -40,31 +40,52 @@ fi
|
|||
|
||||
set -xe
|
||||
|
||||
FWDIR=$(dirname "${BASH_SOURCE-$0}")
|
||||
TIMEOUT_SEC=590
|
||||
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
|
||||
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
|
||||
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
|
||||
|
||||
SPARK_CACHE=".spark-dist"
|
||||
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
|
||||
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
|
||||
echo "SPARK_HOME is ${SPARK_HOME}"
|
||||
if [ ! -d "${SPARK_HOME}" ]; then
|
||||
if [ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
|
||||
# spark 1.1.x and spark 1.2.x can be downloaded from archive
|
||||
STARTTIME=`date +%s`
|
||||
timeout -s KILL 300 wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
|
||||
ENDTIME=`date +%s`
|
||||
DOWNLOADTIME=$((ENDTIME-STARTTIME))
|
||||
else
|
||||
# spark 1.3.x and later can be downloaded from mirror
|
||||
# get download address from mirror
|
||||
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
|
||||
|
||||
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
|
||||
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
|
||||
if [[ ! -d "${SPARK_HOME}" ]]; then
|
||||
mkdir -p "${SPARK_CACHE}"
|
||||
cd "${SPARK_CACHE}"
|
||||
if [[ ! -f "${SPARK_ARCHIVE}.tgz" ]]; then
|
||||
pwd
|
||||
ls -la .
|
||||
echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..."
|
||||
# download archive if not cached
|
||||
if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then
|
||||
# spark 1.1.x and spark 1.2.x can be downloaded from archive
|
||||
STARTTIME=`date +%s`
|
||||
timeout -s KILL "${TIMEOUT_SEC}" wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
|
||||
ENDTIME=`date +%s`
|
||||
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
|
||||
else
|
||||
# spark 1.3.x and later can be downloaded from mirror
|
||||
# get download address from mirror
|
||||
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz?asjson=1")
|
||||
|
||||
STARTTIME=`date +%s`
|
||||
timeout -s KILL 590 wget -q "${PREFFERED}${PATHINFO}"
|
||||
ENDTIME=`date +%s`
|
||||
DOWNLOADTIME=$((ENDTIME-STARTTIME))
|
||||
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
|
||||
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
|
||||
|
||||
STARTTIME=`date +%s`
|
||||
timeout -s KILL "${TIMEOUT_SEC}" wget -q "${PREFFERED}${PATHINFO}"
|
||||
ENDTIME=`date +%s`
|
||||
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
|
||||
fi
|
||||
fi
|
||||
|
||||
# extract archive in un-cached root, clean-up on failure
|
||||
cp "${SPARK_ARCHIVE}.tgz" ..
|
||||
cd ..
|
||||
if ! tar zxf "${SPARK_ARCHIVE}.tgz" ; then
|
||||
echo "Unable to extract ${SPARK_ARCHIVE}.tgz" >&2
|
||||
rm -rf "${SPARK_ARCHIVE}"
|
||||
rm -f "${SPARK_ARCHIVE}.tgz"
|
||||
fi
|
||||
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
|
||||
fi
|
||||
|
||||
set +xe
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
#
|
||||
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
if [[ "$#" -ne 2 ]]; then
|
||||
echo "usage) $0 [spark version] [hadoop version]"
|
||||
echo " eg) $0 1.3.1 2.6"
|
||||
exit 1
|
||||
|
|
@ -26,10 +26,10 @@ fi
|
|||
SPARK_VERSION="${1}"
|
||||
HADOOP_VERSION="${2}"
|
||||
|
||||
echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
if [[ "$?" -eq 0 ]]; then
|
||||
SPARK_VER_RANGE="<=1.2"
|
||||
else
|
||||
SPARK_VER_RANGE="<=1.3"
|
||||
|
|
@ -38,17 +38,18 @@ else
|
|||
SPARK_VER_RANGE=">1.3"
|
||||
fi
|
||||
|
||||
|
||||
set -xe
|
||||
|
||||
FWDIR=$(dirname "${BASH_SOURCE-$0}")
|
||||
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
|
||||
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
|
||||
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
|
||||
|
||||
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
|
||||
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
|
||||
echo "SPARK_HOME is ${SPARK_HOME}"
|
||||
|
||||
# create PID dir. test case detect pid file so they can select active spark home dir for test
|
||||
mkdir -p ${SPARK_HOME}/run
|
||||
export SPARK_PID_DIR=${SPARK_HOME}/run
|
||||
export SPARK_PID_DIR="${SPARK_HOME}/run"
|
||||
mkdir -p "${SPARK_PID_DIR}"
|
||||
|
||||
# start
|
||||
export SPARK_MASTER_PORT=7071
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
if [[ "$#" -ne 2 ]]; then
|
||||
echo "usage) $0 [spark version] [hadoop version]"
|
||||
echo " eg) $0 1.3.1 2.6"
|
||||
exit 1
|
||||
|
|
@ -27,12 +27,15 @@ HADOOP_VERSION="${2}"
|
|||
|
||||
set -xe
|
||||
|
||||
FWDIR=$(dirname "${BASH_SOURCE-$0}")
|
||||
FWDIR="$(dirname "${BASH_SOURCE-$0}")"
|
||||
ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
|
||||
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
|
||||
|
||||
SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
|
||||
export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
|
||||
echo "SPARK_HOME is ${SPARK_HOME}"
|
||||
|
||||
# set create PID dir
|
||||
export SPARK_PID_DIR=${SPARK_HOME}/run
|
||||
export SPARK_PID_DIR="${SPARK_HOME}/run"
|
||||
|
||||
${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
|
||||
${SPARK_HOME}/sbin/stop-master.sh
|
||||
|
|
|
|||
|
|
@ -369,8 +369,8 @@
|
|||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>1.6</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>start-zeppelin</id>
|
||||
|
|
|
|||
|
|
@ -47,10 +47,10 @@
|
|||
<webXml>dist\WEB-INF\web.xml</webXml>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.rat</groupId>
|
||||
<artifactId>apache-rat-plugin</artifactId>
|
||||
<version>0.11</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/.idea/</exclude>
|
||||
|
|
|
|||
|
|
@ -48,10 +48,6 @@
|
|||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.paragraph p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.paragraph div svg {
|
||||
width: 100%;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue