Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'upstream/master' into ZEPPELIN-2365

Browse source 
# Conflicts:
#	zeppelin-web/src/app/home/notebook-template.html
This commit is contained in:
Tinkoff DWH 2017-04-12 09:06:07 +05:00
commit e870ad48d5
50 changed files with 867 additions and 585 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.travis.yml
View file

@ -59,19 +59,19 @@ matrix:
# Test spark module for 2.1.0 with scala 2.11, livy
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
# Test spark module for 2.0.2 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.10
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Psparkr -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.6.3 with scala 2.11
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test python/pyspark with python 2
- jdk: "oraclejdk7"

4
dev/create_release.sh
View file

@ -106,8 +106,8 @@ function make_binary_release() {
git_clone
make_source_package
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pscala-${SCALA_VERSION}"
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
# remove non release files and dirs
rm -rf "${WORKING_DIR}/zeppelin"

2
dev/publish_release.sh
View file

@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
DO_SNAPSHOT="yes"
fi
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Psparkr -Pr"
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"

21
docs/install/build.md
View file

@ -69,7 +69,7 @@ If you're unsure about the options, use the same commands that creates official
# update all pom.xml to use scala 2.11
./dev/change_scala_version.sh 2.11
# build zeppelin with all interpreters and include latest version of Apache spark support for local mode.
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Psparkr -Pr -Pscala-2.11
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pr -Pscala-2.11
```
####3. Done
@ -140,19 +140,10 @@ Available profiles are
-Pscala-2.11
```
##### `-Pyarn` (optional)
enable YARN support for local mode
> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead.
##### `-Pr` (optional)
enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration.
##### `-Psparkr` (optional)
another [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration as well as local mode support.
##### `-Pvendor-repo` (optional)
enable 3rd party vendor repository (cloudera)
@ -184,14 +175,14 @@ Here are some examples with several options:
```bash
# build with spark-2.1, scala-2.11
./dev/change_scala_version.sh 2.11
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Psparkr -Pscala-2.11 -DskipTests
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pscala-2.11 -DskipTests
# build with spark-2.0, scala-2.11
./dev/change_scala_version.sh 2.11
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Psparkr -Pscala-2.11 -DskipTests
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pscala-2.11 -DskipTests
# build with spark-1.6, scala-2.10
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Psparkr -DskipTests
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests -DskipTests
@ -324,10 +315,10 @@ mvn clean package -Pbuild-distr
To build a distribution with specific profiles, run:
```sh
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4
```
The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.
The profiles `-Pspark-1.5 -Phadoop-2.4` can be adjusted if you wish to build to a specific spark versions.
The archive is generated under _`zeppelin-distribution/target`_ directory

2
docs/install/virtual_machine.md
View file

@ -110,7 +110,7 @@ This assumes you've already cloned the project either on the host machine in the
```
cd /zeppelin
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
./bin/zeppelin-daemon.sh start
```

4
docs/interpreter/jdbc.md
View file

@ -164,6 +164,10 @@ There are more JDBC interpreter properties you can specify like below.
<td>zeppelin.jdbc.keytab.location</td>
<td>The path to the keytab file</td>
</tr>
<tr>
<td>zeppelin.jdbc.auth.kerberos.proxy.enable</td>
     <td>When auth type is Kerberos, enable/disable Kerberos proxy with the login user to get the connection. Default value is true.</td>
</tr>
<tr>
<td>default.jceks.file</td>
<td>jceks store path (e.g: jceks://file/tmp/zeppelin.jceks)</td>

4
geode/pom.xml
View file

@ -34,7 +34,7 @@
<properties>
<!--library versions-->
<geode.version>1.0.0-incubating-SNAPSHOT</geode.version>
<geode.version>1.1.0</geode.version>
<commons.exec.version>1.3</commons.exec.version>
</properties>
@ -48,7 +48,7 @@
<dependency>
<groupId>org.apache.geode</groupId>
<artifactId>gemfire-core</artifactId>
<artifactId>geode-core</artifactId>
<version>${geode.version}</version>
</dependency>

12
geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java
View file

@ -29,12 +29,12 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.gemstone.gemfire.cache.client.ClientCache;
import com.gemstone.gemfire.cache.client.ClientCacheFactory;
import com.gemstone.gemfire.cache.query.QueryService;
import com.gemstone.gemfire.cache.query.SelectResults;
import com.gemstone.gemfire.cache.query.Struct;
import com.gemstone.gemfire.pdx.PdxInstance;
import org.apache.geode.cache.client.ClientCache;
import org.apache.geode.cache.client.ClientCacheFactory;
import org.apache.geode.cache.query.QueryService;
import org.apache.geode.cache.query.SelectResults;
import org.apache.geode.cache.query.Struct;
import org.apache.geode.pdx.PdxInstance;
/**
* Apache Geode OQL Interpreter (http://geode.apache.org)

16
geode/src/test/java/org/apache/zeppelin/geode/GeodeOqlInterpreterTest.java
View file

@ -37,14 +37,14 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.Test;
import com.gemstone.gemfire.cache.query.QueryService;
import com.gemstone.gemfire.cache.query.SelectResults;
import com.gemstone.gemfire.cache.query.Struct;
import com.gemstone.gemfire.cache.query.internal.StructImpl;
import com.gemstone.gemfire.cache.query.internal.types.StructTypeImpl;
import com.gemstone.gemfire.pdx.PdxInstance;
import com.gemstone.gemfire.pdx.internal.PdxInstanceImpl;
import com.gemstone.gemfire.pdx.internal.PdxType;
import org.apache.geode.cache.query.QueryService;
import org.apache.geode.cache.query.SelectResults;
import org.apache.geode.cache.query.Struct;
import org.apache.geode.cache.query.internal.StructImpl;
import org.apache.geode.cache.query.internal.types.StructTypeImpl;
import org.apache.geode.pdx.PdxInstance;
import org.apache.geode.pdx.internal.PdxInstanceImpl;
import org.apache.geode.pdx.internal.PdxType;
public class GeodeOqlInterpreterTest {

49
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -372,7 +372,8 @@ public class JDBCInterpreter extends Interpreter {
switch (authType) {
case KERBEROS:
if (user == null) {
if (user == null || "false".equalsIgnoreCase(
property.getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
connection = getConnectionFromPool(url, user, propertyKey, properties);
} else {
if (url.trim().startsWith("jdbc:hive")) {
@ -507,19 +508,36 @@ public class JDBCInterpreter extends Interpreter {
protected ArrayList<String> splitSqlQueries(String sql) {
ArrayList<String> queries = new ArrayList<>();
StringBuilder query = new StringBuilder();
Character character;
char character;
Boolean antiSlash = false;
Boolean multiLineComment = false;
Boolean singleLineComment = false;
Boolean quoteString = false;
Boolean doubleQuoteString = false;
for (int item = 0; item < sql.length(); item++) {
character = sql.charAt(item);
if (character.equals('\\')) {
if ((singleLineComment && (character == '\n' || item == sql.length() - 1))
|| (multiLineComment && character == '/' && sql.charAt(item - 1) == '*')) {
singleLineComment = false;
multiLineComment = false;
if (item == sql.length() - 1 && query.length() > 0) {
queries.add(StringUtils.trim(query.toString()));
}
continue;
}
if (singleLineComment || multiLineComment) {
continue;
}
if (character == '\\') {
antiSlash = true;
}
if (character.equals('\'')) {
if (character == '\'') {
if (antiSlash) {
antiSlash = false;
} else if (quoteString) {
@ -528,7 +546,8 @@ public class JDBCInterpreter extends Interpreter {
quoteString = true;
}
}
if (character.equals('"')) {
if (character == '"') {
if (antiSlash) {
antiSlash = false;
} else if (doubleQuoteString) {
@ -538,16 +557,30 @@ public class JDBCInterpreter extends Interpreter {
}
}
if (character.equals(';') && !antiSlash && !quoteString && !doubleQuoteString) {
queries.add(query.toString());
if (!quoteString && !doubleQuoteString && !multiLineComment && !singleLineComment
&& sql.length() > item + 1) {
if (character == '-' && sql.charAt(item + 1) == '-') {
singleLineComment = true;
continue;
}
if (character == '/' && sql.charAt(item + 1) == '*') {
multiLineComment = true;
continue;
}
}
if (character == ';' && !antiSlash && !quoteString && !doubleQuoteString) {
queries.add(StringUtils.trim(query.toString()));
query = new StringBuilder();
} else if (item == sql.length() - 1) {
query.append(character);
queries.add(query.toString());
queries.add(StringUtils.trim(query.toString()));
} else {
query.append(character);
}
}
return queries;
}

28
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -450,4 +450,32 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
}
@Test
public void testExcludingComments() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "/* ; */\n" +
"-- /* comment\n" +
"--select * from test_table\n" +
"select * from test_table; /* some comment ; */\n" +
"/*\n" +
"select * from test_table;\n" +
"*/\n" +
"-- a ; b\n" +
"select * from test_table WHERE ID = ';--';\n" +
"select * from test_table WHERE ID = '/*' -- test";
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(3, interpreterResult.message().size());
}
}

5
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
View file

@ -25,6 +25,7 @@ import org.apache.http.client.HttpClient;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.impl.client.HttpClients;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.interpreter.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -369,13 +370,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
}
if (displayAppInfo) {
//TODO(zjffdu), use multiple InterpreterResult to display appInfo
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
interpreterResult.add(InterpreterResult.Type.TEXT, result);
String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
+ "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
+ sessionInfo.webUIAddress + "</a>";
LOGGER.info("appInfoHtml:" + appInfoHtml);
interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml);
return interpreterResult;
} else {
@ -486,6 +485,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
if (cause.getResponseBodyAsString().matches(SESSION_NOT_FOUND_PATTERN)) {
throw new SessionNotFoundException(cause.getResponseBodyAsString());
}
throw new LivyException(cause.getResponseBodyAsString() + "\n"
+ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e)));
}
throw new LivyException(e);
}

2
notebook/2C2AUG798/note.json
View file

@ -40,7 +40,7 @@
"msg": [
{
"type": "HTML",
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this tutorial we will go through some of the basic features of Zeppelin\u0026rsquo;s built-in matplotlib integration. \u003c/p\u003e\n\u003ch3\u003ePrerequisites\u003c/h3\u003e\n\u003cp\u003e\u003ccode\u003ematplotlib\u003c/code\u003e must be installed to your local python installation. (use \u003ccode\u003epip install matplotlib\u003c/code\u003e or \u003ccode\u003econda install matplotlib\u003c/code\u003e if you have \u003ccode\u003econda\u003c/code\u003e). Additionally, you will need Zeppelin\u0026rsquo;s matplotlib backend files which are usually found in \u003ccode\u003e$ZEPPELIN_HOME/lib/python\u003c/code\u003e. Although Zeppelin should automatically find this directory, it might be a good idea to add it to your \u003ccode\u003ePYTHONPATH\u003c/code\u003e just in case. \u003c/p\u003e\n\u003ch3\u003eInterpreters\u003c/h3\u003e\n\u003cp\u003eMost of the examples shown in this tutorial can be used interchangeably with either the \u003ccode\u003epython\u003c/code\u003e or \u003ccode\u003epyspark\u003c/code\u003e interpreters. Iterative plotting using the Angular Display System is currently only available for \u003ccode\u003epyspark\u003c/code\u003e, but this functionality will eventually be added to the base \u003ccode\u003epython\u003c/code\u003e interpreter. \u003c/p\u003e\n\u003ch3\u003emacOS\u003c/h3\u003e\n\u003cp\u003eMake sure locale is set, to avoid \u003ccode\u003eValueError: unknown locale: UTF-8\u003c/code\u003e\u003c/p\u003e\n\u003ch3\u003evirtualenv\u003c/h3\u003e\n\u003cp\u003eIn case you want to use virtualenv or conda env:\u003cbr/\u003e - configure python interpreter property -\u0026gt; \u003ccode\u003eabsolute/path/to/venv/bin/python\u003c/code\u003e\u003cbr/\u003e - see \u003cem\u003eWorking with Matplotlib in Virtual environments\u003c/em\u003e in the \u003ca href\u003d\"http://matplotlib.org/faq/virtualenv_faq.html\"\u003eMatplotlib FAQ\u003c/a\u003e\u003c/p\u003e\n\u003ch3\u003eA simple example\u003c/h3\u003e\n\u003cp\u003eLet\u0026rsquo;s start by making a very simple line plot:\u003c/p\u003e\n\u003c/div\u003e"
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this tutorial we will go through some of the basic features of Zeppelin\u0026rsquo;s built-in matplotlib integration. \u003c/p\u003e\n\u003ch3\u003ePrerequisites\u003c/h3\u003e\n\u003cp\u003e\u003ccode\u003ematplotlib\u003c/code\u003e must be installed to your local python installation. (use \u003ccode\u003epip install matplotlib\u003c/code\u003e or \u003ccode\u003econda install matplotlib\u003c/code\u003e if you have \u003ccode\u003econda\u003c/code\u003e). Additionally, you will need Zeppelin\u0026rsquo;s matplotlib backend files which are usually found in \u003ccode\u003e$ZEPPELIN_HOME/interpreter/lib/python\u003c/code\u003e. Although Zeppelin should automatically find this directory, it might be a good idea to add it to your \u003ccode\u003ePYTHONPATH\u003c/code\u003e just in case. \u003c/p\u003e\n\u003ch3\u003eInterpreters\u003c/h3\u003e\n\u003cp\u003eMost of the examples shown in this tutorial can be used interchangeably with either the \u003ccode\u003epython\u003c/code\u003e or \u003ccode\u003epyspark\u003c/code\u003e interpreters. Iterative plotting using the Angular Display System is currently only available for \u003ccode\u003epyspark\u003c/code\u003e, but this functionality will eventually be added to the base \u003ccode\u003epython\u003c/code\u003e interpreter. \u003c/p\u003e\n\u003ch3\u003emacOS\u003c/h3\u003e\n\u003cp\u003eMake sure locale is set, to avoid \u003ccode\u003eValueError: unknown locale: UTF-8\u003c/code\u003e\u003c/p\u003e\n\u003ch3\u003evirtualenv\u003c/h3\u003e\n\u003cp\u003eIn case you want to use virtualenv or conda env:\u003cbr/\u003e - configure python interpreter property -\u0026gt; \u003ccode\u003eabsolute/path/to/venv/bin/python\u003c/code\u003e\u003cbr/\u003e - see \u003cem\u003eWorking with Matplotlib in Virtual environments\u003c/em\u003e in the \u003ca href\u003d\"http://matplotlib.org/faq/virtualenv_faq.html\"\u003eMatplotlib FAQ\u003c/a\u003e\u003c/p\u003e\n\u003ch3\u003eA simple example\u003c/h3\u003e\n\u003cp\u003eLet\u0026rsquo;s start by making a very simple line plot:\u003c/p\u003e\n\u003c/div\u003e"
}
]
},

2
scripts/vagrant/zeppelin-dev/README.md
View file

@ -87,7 +87,7 @@ This assumes you've already cloned the project either on the host machine in the
```
cd /zeppelin
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
./bin/zeppelin-daemon.sh start
```

2
scripts/vagrant/zeppelin-dev/show-instructions.sh
View file

@ -34,7 +34,7 @@ echo 'mvn clean package -DskipTests'
echo
echo '# or for a specific Spark/Hadoop build with additional options such as python and R support'
echo
echo 'mvn clean package -Pspark-1.6 -Phadoop-2.4 -Psparkr -DskipTests'
echo 'mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests'
echo './bin/zeppelin-daemon.sh start'
echo
echo 'On your host machine browse to http://localhost:8080/'

126
spark-dependencies/pom.xml
View file

@ -344,6 +344,19 @@
</exclusions>
</dependency>
<!-- yarn (not supported for Spark v1.5.0 or higher) -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
</dependencies>
<profiles>
@ -810,74 +823,6 @@
</repositories>
</profile>
<profile>
<id>yarn</id>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>sparkr</id>
<build>
<plugins>
<plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<artifactId>download-maven-plugin</artifactId>
<executions>
<execution>
<id>download-sparkr-files</id>
<phase>validate</phase>
<goals>
<goal>wget</goal>
</goals>
<configuration>
<readTimeOut>60000</readTimeOut>
<retries>5</retries>
<url>${spark.bin.download.url}</url>
<unpack>true</unpack>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
<executions>
<execution>
<id>copy-sparkr-files</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/spark/R/lib</outputDirectory>
<resources>
<resource>
<directory>
${project.build.directory}/spark-${spark.version}-bin-without-hadoop/R/lib
</directory>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
@ -1039,6 +984,51 @@
</execution>
</executions>
</plugin>
<!-- include sparkr by default -->
<plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<artifactId>download-maven-plugin</artifactId>
<executions>
<execution>
<id>download-sparkr-files</id>
<phase>validate</phase>
<goals>
<goal>wget</goal>
</goals>
<configuration>
<readTimeOut>60000</readTimeOut>
<retries>5</retries>
<url>${spark.bin.download.url}</url>
<unpack>true</unpack>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
<executions>
<execution>
<id>copy-sparkr-files</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/spark/R/lib</outputDirectory>
<resources>
<resource>
<directory>
${project.build.directory}/spark-${spark.version}-bin-without-hadoop/R/lib
</directory>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

109
spark/pom.xml
View file

@ -320,6 +320,19 @@
</dependencies>
<build>
<!-- sparkr resources -->
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>interpreter-setting.json</exclude>
</excludes>
</resource>
<resource>
<directory>src/main/sparkr-resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
@ -440,18 +453,13 @@
</executions>
</plugin>
<!-- exclude sparkr by default. sparkr is enabled by profile 'sparkr' -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/SparkRInterpreter.java</exclude>
</excludes>
<testExcludes>
<excludes combine.self="override"></excludes>
<testExcludes combine.self="override">
<testExclude>${pyspark.test.exclude}</testExclude>
<testExclude>**/SparkRInterpreterTest.java</testExclude>
<testExclude>**/ZeppelinRTest.java</testExclude>
</testExcludes>
</configuration>
</plugin>
@ -459,9 +467,7 @@
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/ZeppelinR.scala</exclude>
<exclude>**/SparkRBackend.scala</exclude>
<excludes combine.self="override">
</excludes>
</configuration>
</plugin>
@ -469,12 +475,37 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<includes>
<include>${pyspark.test.include}</include>
</includes>
<excludes>
<excludes combine.self="override">
<exclude>${pyspark.test.exclude}</exclude>
</excludes>
</configuration>
</plugin>
<!-- include sparkr by default -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes combine.self="override"></excludes>
<testExcludes combine.self="override">
<testExclude>${pyspark.test.exclude}</testExclude>
</testExcludes>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<configuration>
<excludes combine.self="override">
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<excludes combine.self="override">
<exclude>${pyspark.test.exclude}</exclude>
<exclude>**/SparkRInterpreterTest.java</exclude>
</excludes>
</configuration>
</plugin>
@ -609,53 +640,5 @@
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
<!-- include sparkr in the build -->
<profile>
<id>sparkr</id>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>interpreter-setting.json</exclude>
</excludes>
</resource>
<resource>
<directory>src/main/sparkr-resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes combine.self="override"></excludes>
<testExcludes combine.self="override">
<testExclude>${pyspark.test.exclude}</testExclude>
</testExcludes>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<configuration>
<excludes combine.self="override">
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<excludes combine.self="override">
<exclude>${pyspark.test.exclude}</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

17
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -999,10 +999,19 @@ public class SparkInterpreter extends Interpreter {
if (sparkUrl != null) {
return sparkUrl;
}
Option<SparkUI> sparkUiOption = (Option<SparkUI>) Utils.invokeMethod(sc, "ui");
SparkUI sparkUi = sparkUiOption.get();
String sparkWebUrl = sparkUi.appUIAddress();
return sparkWebUrl;
if (sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0)) {
Option<String> uiWebUrlOption = (Option<String>) Utils.invokeMethod(sc, "uiWebUrl");
if (uiWebUrlOption.isDefined()) {
return uiWebUrlOption.get();
}
} else {
Option<SparkUI> sparkUIOption = (Option<SparkUI>) Utils.invokeMethod(sc, "ui");
if (sparkUIOption.isDefined()) {
return (String) Utils.invokeMethod(sparkUIOption.get(), "appUIAddress");
}
}
return null;
}
private Results.Result interpret(String line) {

4
testing/install_external_dependencies.sh
View file

@ -20,8 +20,8 @@
set -ev
touch ~/.environ
# Install R dependencies if R profiles are used
if [[ ${PROFILE/"-Pr "} != $PROFILE ]] || [[ ${PROFILE/"-Psparkr "} != $PROFILE ]] ; then
# Install R dependencies if SPARKR is true
if [[ "${SPARKR}" = "true" ]] ; then
echo "R_LIBS=~/R" > ~/.Renviron
echo "export R_LIBS=~/R" >> ~/.environ
source ~/.environ

2
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -229,7 +229,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) Json3 v3.3.1 (http://bestiejs.github.io/json3) - https://github.com/bestiejs/json3/blob/v3.3.1/LICENSE
(The MIT License) es5-shim v3.1.0 (https://github.com/es-shims/es5-shim) - https://github.com/es-shims/es5-shim/blob/v3.1.0/LICENSE
(The MIT License) bootstrap v3.2.0 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.2.0/LICENSE
(The MIT License) UI Bootstrap v0.13.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/0.13.0/LICENSE
(The MIT License) UI Bootstrap v2.5.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/2.5.0/LICENSE
(The MIT License) bootstrap3-dialog v1.34.7 (https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7) - https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7
(The MIT License) Angular Websocket v1.0.13 (http://angularclass.github.io/angular-websocket/) - https://github.com/AngularClass/angular-websocket/blob/v1.0.13/LICENSE
(The MIT License) UI.Ace v0.1.1 (http://angularclass.github.io/angular-websocket/) - https://github.com/angular-ui/ui-ace/blob/master/LICENSE

50
zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
View file

@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.helium.HeliumPackageSearchResult;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
@ -68,6 +69,16 @@ public class HeliumRestApi {
Response.Status.OK, "", helium.getAllPackageInfo()).build();
}
/**
* Get all enabled package infos
*/
@GET
@Path("enabledPackage")
public Response getAllEnabledPackageInfo() {
return new JsonResponse(
Response.Status.OK, "", helium.getAllEnabledPackages()).build();
}
/**
* Get single package info
*/
@ -130,26 +141,43 @@ public class HeliumRestApi {
}
@GET
@Path("bundle/load")
@Path("bundle/load/{packageName}")
@Produces("text/javascript")
public Response bundleLoad(@QueryParam("refresh") String refresh) {
public Response bundleLoad(@QueryParam("refresh") String refresh,
@PathParam("packageName") String packageName) {
if (StringUtils.isEmpty(packageName)) {
return new JsonResponse(
Response.Status.BAD_REQUEST,
"Can't get bundle due to empty package name").build();
}
HeliumPackageSearchResult psr = null;
List<HeliumPackageSearchResult> enabledPackages = helium.getAllEnabledPackages();
for (HeliumPackageSearchResult e : enabledPackages) {
if (e.getPkg().getName().equals(packageName)) {
psr = e;
break;
}
}
if (psr == null) {
// return empty to specify
return Response.ok().build();
}
try {
File bundle;
if (refresh != null && refresh.equals("true")) {
bundle = helium.recreateBundle();
} else {
bundle = helium.getBundleFactory().getCurrentCacheBundle();
}
boolean rebuild = (refresh != null && refresh.equals("true"));
bundle = helium.getBundle(psr.getPkg(), rebuild);
if (bundle == null) {
return Response.ok().build();
} else {
String stringifiedBundle = FileUtils.readFileToString(bundle);
return Response.ok(stringifiedBundle).build();
String stringified = FileUtils.readFileToString(bundle);
return Response.ok(stringified).build();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
// returning error will prevent zeppelin front-end render any notebook.
// visualization load fail doesn't need to block notebook rendering work.
// so it's better return ok instead of any error.
@ -172,7 +200,7 @@ public class HeliumRestApi {
@POST
@Path("disable/{packageName}")
public Response enablePackage(@PathParam("packageName") String packageName) {
public Response disablePackage(@PathParam("packageName") String packageName) {
try {
helium.disable(packageName);
return new JsonResponse(Response.Status.OK).build();

4
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -115,6 +115,7 @@ public class ZeppelinServer extends Application {
*/
heliumBundleFactory = new HeliumBundleFactory(
conf,
null,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
@ -122,6 +123,7 @@ public class ZeppelinServer extends Application {
} else {
heliumBundleFactory = new HeliumBundleFactory(
conf,
null,
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")),
new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")),
@ -138,7 +140,7 @@ public class ZeppelinServer extends Application {
// create bundle
try {
heliumBundleFactory.buildBundle(helium.getBundlePackagesToBundle());
heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}

2
zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java
View file

@ -125,7 +125,7 @@ public class WebDriverManager {
(new WebDriverWait(driver, 30)).until(new ExpectedCondition<Boolean>() {
@Override
public Boolean apply(WebDriver d) {
return d.findElement(By.xpath("//i[@tooltip='WebSocket Connected']"))
return d.findElement(By.xpath("//i[@uib-tooltip='WebSocket Connected']"))
.isDisplayed();
}
});

2
zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
View file

@ -187,7 +187,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
String noteId = driver.getCurrentUrl().substring(driver.getCurrentUrl().lastIndexOf("/") + 1);
pollingWait(By.xpath("//span[@tooltip='Note permissions']"),
pollingWait(By.xpath("//span[@uib-tooltip='Note permissions']"),
MAX_BROWSER_TIMEOUT_SEC).click();
pollingWait(By.xpath(".//*[@id='selectOwners']/following::span//input"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("finance ");

4
zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
View file

@ -198,8 +198,8 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
}
try {
// restart spark interpreter before running %dep
clickAndWait(By.xpath("//span[@tooltip='Interpreter binding']"));
clickAndWait(By.xpath("//div[font[contains(text(), 'spark')]]/preceding-sibling::a[@tooltip='Restart']"));
clickAndWait(By.xpath("//span[@uib-tooltip='Interpreter binding']"));
clickAndWait(By.xpath("//div[font[contains(text(), 'spark')]]/preceding-sibling::a[@uib-tooltip='Restart']"));
clickAndWait(By.xpath("//button[contains(.,'OK')]"));
setTextOfParagraph(1,"%dep z.load(\"org.apache.commons:commons-csv:1.1\")");

2
zeppelin-web/bower.json
View file

@ -12,7 +12,7 @@
"angular-touch": "1.5.0",
"angular-route": "1.5.0",
"angular-resource": "1.5.0",
"angular-bootstrap": "~0.13.0",
"angular-bootstrap": "~2.5.0",
"angular-websocket": "~1.0.13",
"ace-builds": "1.2.6",
"angular-ui-ace": "0.1.3",

2
zeppelin-web/src/app/credential/credential.html
View file

@ -22,7 +22,7 @@ limitations under the License.
<a style="cursor:pointer;margin-right:10px;text-decoration:none;"
target="_blank"
ng-href="http://zeppelin.apache.org/docs/{{zeppelinVersion}}/security/datasource_authorization.html"
tooltip-placement="bottom" tooltip="Learn more">
tooltip-placement="bottom" uib-tooltip="Learn more">
<i class="icon-question" ng-style="{color: showRepositoryInfo ? '#3071A9' : 'black' }"></i>
</a>
<button class="btn btn-default btn-sm"

7
zeppelin-web/src/app/helium/helium.html
View file

@ -23,8 +23,7 @@ limitations under the License.
<a target="_blank"
class="helium-repo-btn"
ng-href="https://zeppelin.apache.org/helium_packages.html"
tooltip-placement="bottom"
tooltip="Learn more">
tooltip-placement="bottom" uib-tooltip="Learn more">
<i class="icon-question" ng-style="{color: 'black'}"></i>
</a>
<button tabindex="0" role="button"
@ -186,13 +185,13 @@ limitations under the License.
</div>
</div>
<div class="text-center" style="margin-top: 20px;" ng-if="getPackageSize(defaultPackages, pkgListByType) !== 0">
<pagination boundary-links="true" total-items="defaultPackages.length"
<ul uib-pagination boundary-links="true" total-items="defaultPackages.length"
ng-model="$parent.currentPage" class="pagination-sm"
ng-show="$parent.pkgListByType === types"
max-size="maxSize"
items-per-page="itemsPerPage"
ng-class="{'hide-first-boundaries': $parent.currentPage == 1, 'hide-last-boundaries': $parent.currentPage >= defaultPackages.length/itemsPerPage}"
previous-text="&lsaquo;" next-text="&rsaquo;" first-text="&laquo;" last-text="&raquo;"></pagination>
previous-text="&lsaquo;" next-text="&rsaquo;" first-text="&laquo;" last-text="&raquo;"></ul>
</div>
</div>
</div>

2
zeppelin-web/src/app/home/home.html
View file

@ -29,7 +29,7 @@ limitations under the License.
<i ng-class="isReloadingNotes ? 'fa fa-refresh fa-spin' : 'fa fa-refresh'"
ng-style="!isReloadingNotes && {'cursor': 'pointer'}" style="font-size: 13px;"
ng-click="reloadNoteList();"
tooltip-placement="bottom" tooltip="Reload notes from storage">
tooltip-placement="bottom" uib-tooltip="Reload notes from storage">
</i>
</h4>

24
zeppelin-web/src/app/home/notebook-template.html
View file

@ -25,28 +25,28 @@ limitations under the License.
<i style="margin-left: 10px;"
class="fa fa-pencil notebook-list-btn" ng-show="showNoteButton"
ng-click="node.path ? renameNote(node.id, node.path) : renameNote(node.id, node.name)"
tooltip-placement="bottom" tooltip="Rename note">
tooltip-placement="bottom" uib-tooltip="Rename note">
</i>
</a>
<a ng-if="!node.isTrash" style="text-decoration: none;">
<i class="fa fa-eraser notebook-list-btn" ng-show="showNoteButton" ng-click="clearAllParagraphOutput(node.id)"
tooltip-placement="bottom" tooltip="Clear output">
tooltip-placement="bottom" uib-tooltip="Clear output">
</i>
</a>
<a ng-if="!node.isTrash" style="text-decoration: none;">
<i class="fa fa-trash-o notebook-list-btn" ng-show="showNoteButton" ng-click="moveNoteToTrash(node.id)"
tooltip-placement="bottom" tooltip="Move note to Trash">
tooltip-placement="bottom" uib-tooltip="Move note to Trash">
</i>
</a>
<!-- if note is in trash -->
<a ng-if="node.isTrash">
<i class="fa fa-undo notebook-list-btn" ng-show="showNoteButton" ng-click="restoreNote(node.id)"
tooltip-placement="bottom" tooltip="Restore note">
tooltip-placement="bottom" uib-tooltip="Restore note">
</i>
</a>
<a ng-if="node.isTrash" style="font-size: 16px;">
<i class="fa fa-times notebook-list-btn" ng-show="showNoteButton" ng-click="removeNote(node.id)"
tooltip-placement="bottom" tooltip="Remove note permanently">
tooltip-placement="bottom" uib-tooltip="Remove note permanently">
</i>
</a>
</div>
@ -63,30 +63,30 @@ limitations under the License.
ng-controller="NotenameCtrl as notenamectrl" ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}">
<i style="margin-left: 10px;"
class="fa fa-plus notebook-list-btn" ng-show="showFolderButton"
tooltip-placement="bottom" tooltip="Create new note">
tooltip-placement="bottom" uib-tooltip="Create new note">
</i>
</a>
</a>
<a ng-if="!node.isTrash" style="text-decoration: none;">
<i class="fa fa-pencil notebook-list-btn" ng-show="showFolderButton" ng-click="renameFolder(node)"
tooltip-placement="bottom" tooltip="Rename folder">
tooltip-placement="bottom" uib-tooltip="Rename folder">
</i>
</a>
<!-- if folder is not in trash -->
<a ng-if="!node.isTrash">
<i class="fa fa-trash-o notebook-list-btn" ng-show="showFolderButton" ng-click="moveFolderToTrash(node.id)"
tooltip-placement="bottom" tooltip="Move folder to Trash">
tooltip-placement="bottom" uib-tooltip="Move folder to Trash">
</i>
</a>
<!-- if folder is in trash -->
<a ng-if="node.isTrash">
<i class="fa fa-undo notebook-list-btn" ng-show="showFolderButton" ng-click="restoreFolder(node.id)"
tooltip-placement="bottom" tooltip="Restore folder">
tooltip-placement="bottom" uib-tooltip="Restore folder">
</i>
</a>
<a ng-if="node.isTrash" style="font-size: 16px">
<i class="fa fa-times notebook-list-btn" ng-show="showFolderButton" ng-click="removeFolder(node.id)"
tooltip-placement="bottom" tooltip="Remove folder permanently">
tooltip-placement="bottom" uib-tooltip="Remove folder permanently">
</i>
</a>
</div>
@ -106,11 +106,11 @@ limitations under the License.
<a style="text-decoration: none;">
<i style="margin-left: 10px"
class="fa fa-undo notebook-list-btn" ng-show="showFolderButton" ng-click="restoreAll()"
tooltip-placement="bottom" tooltip="Restore all">
tooltip-placement="bottom" uib-tooltip="Restore all">
</i>
<i style="font-size: 16px;"
class="fa fa-times notebook-list-btn" ng-show="showFolderButton" ng-click="emptyTrash()"
tooltip-placement="bottom" tooltip="Empty trash">
tooltip-placement="bottom" uib-tooltip="Empty trash">
</i>
</a>
</div>

2
zeppelin-web/src/app/home/notebook.html
View file

@ -18,7 +18,7 @@ limitations under the License.
<i ng-class="isReloadingNotes ? 'fa fa-refresh fa-spin' : 'fa fa-refresh'"
ng-style="!isReloadingNotes && {'cursor': 'pointer'}" style="font-size: 13px;"
ng-click="reloadNoteList();"
tooltip-placement="bottom" tooltip="Reload notes from storage">
tooltip-placement="bottom" uib-tooltip="Reload notes from storage">
</i>
</h4>
<h5><a href="" data-toggle="modal" data-target="#noteImportModal" style="text-decoration: none;">

14
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -84,7 +84,7 @@ limitations under the License.
<li
ng-if="getInterpreterRunningOption(setting.id) === 'Globally'">
<a style="cursor:pointer"
tooltip="Single interpreter instance are shared across notes"
uib-tooltip="Single interpreter instance are shared across notes"
ng-click="setPerNoteOption(setting.id, 'shared')">
shared per note
</a>
@ -93,7 +93,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
@ -101,7 +101,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerUserOption(setting.id, 'scoped')">
scoped per user
</a>
@ -110,7 +110,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>
@ -118,7 +118,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerUserOption(setting.id, 'isolated')">
isolated per user
</a>
@ -171,14 +171,14 @@ limitations under the License.
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>

20
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -109,18 +109,18 @@ limitations under the License.
<small ng-switch="setting.status">
<small ng-switch-when="READY">
<i style="color: green; margin-right: 3px;" class="fa fa-circle"
tooltip="Ready">
uib-tooltip="Ready">
</i>
</small>
<small ng-switch-when="ERROR">
<i style="color: red; cursor: pointer" class="fa fa-circle"
ng-click="showErrorMessage(setting)"
tooltip="Error downloading dependencies">
uib-tooltip="Error downloading dependencies">
</i>
</small>
<small ng-switch-default>
<i style="color: blue" class="fa fa-spinner spinAnimation"
tooltip="Dependencies are downloading">
uib-tooltip="Dependencies are downloading">
</i>
</small>
</small>
@ -196,7 +196,7 @@ limitations under the License.
<li
ng-if="getInterpreterRunningOption(setting.id) === 'Globally'">
<a style="cursor:pointer"
tooltip="Single interpreter instance are shared across notes"
uib-tooltip="Single interpreter instance are shared across notes"
ng-click="setPerNoteOption(setting.id, 'shared')">
shared per note
</a>
@ -205,7 +205,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
@ -213,7 +213,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerUserOption(setting.id, 'scoped')">
scoped per user
</a>
@ -222,7 +222,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>
@ -230,7 +230,7 @@ limitations under the License.
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerUserOption(setting.id, 'isolated')">
isolated per user
</a>
@ -285,14 +285,14 @@ limitations under the License.
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter instance for each note"
uib-tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter process for each note"
uib-tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>

2
zeppelin-web/src/app/jobmanager/jobmanager.html
View file

@ -39,7 +39,7 @@ limitations under the License.
class="btn btn-default"
style="width: 25px; height: 25px; margin-right: 0px; padding: 1px 0px 3px 3px"
ng-click="onChangeSortAsc()"
tooltip-placement="right" tooltip="{{sortTooltipMsg}}">
tooltip-placement="right" uib-tooltip="{{sortTooltipMsg}}">
<i class="fa" ng-class="{true: 'fa-sort-amount-asc', false : 'fa-sort-amount-desc'}[filterConfig.isSortByAsc]"></i>
</button>
</span>

4
zeppelin-web/src/app/jobmanager/jobs/job-control.html
View file

@ -31,12 +31,12 @@ limitations under the License.
<!-- Run / Cancel button -->
<span
ng-if="!notebookJob.isRunningJob"
class="icon-control-play" style="cursor:pointer;color:#3071A9" tooltip-placement="left" tooltip="START ALL Job"
class="icon-control-play" style="cursor:pointer;color:#3071A9" tooltip-placement="left" uib-tooltip="START ALL Job"
ng-click="runNotebookJob(notebookJob.noteId)">
</span>
<span
ng-if="notebookJob.isRunningJob"
class="icon-control-pause" style="cursor:pointer;color:#3071A9" tooltip-placement="left" tooltip="STOP ALL Job"
class="icon-control-pause" style="cursor:pointer;color:#3071A9" tooltip-placement="left" uib-tooltip="STOP ALL Job"
ng-click="stopNotebookJob(notebookJob.noteId)">
</span>
</div>

14
zeppelin-web/src/app/jobmanager/jobs/job.html
View file

@ -51,7 +51,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: green" class="fa fa-circle-o"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is READY">
uib-tooltip="{{paragraphJob.name}} is READY">
</i>
</a>
<a ng-switch-when="FINISHED"
@ -59,7 +59,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: green" class="fa fa-circle"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is FINISHED">
uib-tooltip="{{paragraphJob.name}} is FINISHED">
</i>
</a>
<a ng-switch-when="ABORT"
@ -67,7 +67,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: orange" class="fa fa-circle"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is ABORT">
uib-tooltip="{{paragraphJob.name}} is ABORT">
</i>
</a>
<a ng-switch-when="ERROR"
@ -75,7 +75,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: red" class="fa fa-circle"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is ERROR">
uib-tooltip="{{paragraphJob.name}} is ERROR">
</i>
</a>
<a ng-switch-when="PENDING"
@ -83,7 +83,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: gray" class="fa fa-circle"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is PENDING">
uib-tooltip="{{paragraphJob.name}} is PENDING">
</i>
</a>
<a ng-switch-when="RUNNING"
@ -91,7 +91,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i style="color: blue" class="fa fa-spinner spinAnimation"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is RUNNING">
uib-tooltip="{{paragraphJob.name}} is RUNNING">
</i>
</a>
<a ng-switch-default class="icon-question"
@ -99,7 +99,7 @@ limitations under the License.
ng-href="#/notebook/{{notebookJob.noteId}}?paragraph={{paragraphJob.id}}">
<i class="icon-question"
tooltip-placement="top-left"
tooltip="{{paragraphJob.name}} is {{paragraphJob.status}}">
uib-tooltip="{{paragraphJob.name}} is {{paragraphJob.status}}">
</i>
</a>
</span>

34
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -26,7 +26,7 @@ limitations under the License.
class="btn btn-default btn-xs"
ng-click="runAllParagraphs(note.id)"
ng-class="{'disabled':isNoteRunning()}"
tooltip-placement="bottom" tooltip="Run all paragraphs"
tooltip-placement="bottom" uib-tooltip="Run all paragraphs"
ng-disabled="revisionView">
<i class="icon-control-play"></i>
</button>
@ -34,14 +34,14 @@ limitations under the License.
class="btn btn-default btn-xs"
ng-click="toggleAllEditor()"
ng-hide="viewOnly"
tooltip-placement="bottom" tooltip="Show/hide the code"
tooltip-placement="bottom" uib-tooltip="Show/hide the code"
ng-disabled="revisionView">
<i ng-class="editorToggled ? 'fa icon-size-fullscreen' :'fa icon-size-actual'"></i></button>
<button type="button"
class="btn btn-default btn-xs"
ng-click="toggleAllTable()"
ng-hide="viewOnly"
tooltip-placement="bottom" tooltip="Show/hide the output"
tooltip-placement="bottom" uib-tooltip="Show/hide the output"
ng-disabled="revisionView">
<i ng-class="tableToggled ? 'fa icon-notebook' : 'fa icon-book-open'"></i>
</button>
@ -50,7 +50,7 @@ limitations under the License.
ng-click="clearAllParagraphOutput(note.id)"
ng-hide="viewOnly"
ng-class="{'disabled':isNoteRunning()}"
tooltip-placement="bottom" tooltip="Clear output"
tooltip-placement="bottom" uib-tooltip="Clear output"
ng-disabled="revisionView">
<i class="fa fa-eraser"></i>
</button>
@ -58,7 +58,7 @@ limitations under the License.
<button type="button"
class="btn btn-default btn-xs"
ng-hide="viewOnly"
tooltip-placement="bottom" tooltip="Clone this note" data-source-note-name="{{note.name}}"
tooltip-placement="bottom" uib-tooltip="Clone this note" data-source-note-name="{{note.name}}"
data-toggle="modal" data-target="#noteNameModal" data-clone="true"
ng-disabled="revisionView">
<i class="fa fa-copy"></i>
@ -67,7 +67,7 @@ limitations under the License.
class="btn btn-default btn-xs"
ng-hide="viewOnly"
ng-click="exportNote()"
tooltip-placement="bottom" tooltip="Export this note"
tooltip-placement="bottom" uib-tooltip="Export this note"
ng-disabled="revisionView">
<i class="fa fa-download"></i>
</button>
@ -78,7 +78,7 @@ limitations under the License.
ng-if="ticket.principal && ticket.principal !== 'anonymous'"
ng-hide="viewOnly || note.config.personalizedMode !== 'true'"
ng-click="toggleNotePersonalizedMode()"
tooltip-placement="bottom" tooltip="Switch to collaboration mode {{isOwner ? '' : '(owner can change)'}}"
tooltip-placement="bottom" uib-tooltip="Switch to collaboration mode {{isOwner ? '' : '(owner can change)'}}"
ng-disabled="revisionView">
<i class="fa fa-user"></i>
</button>
@ -88,7 +88,7 @@ limitations under the License.
ng-if="ticket.principal && ticket.principal !== 'anonymous'"
ng-hide="viewOnly || note.config.personalizedMode === 'true'"
ng-click="toggleNotePersonalizedMode()"
tooltip-placement="bottom" tooltip="Switch to personal mode {{isOwner ? '' : '(owner can change)'}}"
tooltip-placement="bottom" uib-tooltip="Switch to personal mode {{isOwner ? '' : '(owner can change)'}}"
ng-disabled="revisionView">
<i class="fa fa-users"></i>
</button>
@ -101,7 +101,7 @@ limitations under the License.
id="versionControlDropdown"
ng-hide="viewOnly"
data-toggle="dropdown"
tooltip-placement="bottom" tooltip="Version control"
tooltip-placement="bottom" uib-tooltip="Version control"
ng-disabled="revisionView">
<i class="fa fa-file-code-o"></i>
</button>
@ -111,7 +111,7 @@ limitations under the License.
ng-hide="viewOnly"
ng-click="setNoteRevision()"
ng-disabled="!revisionView"
tooltip-placement="bottom" tooltip="Set revision">
tooltip-placement="bottom" uib-tooltip="Set revision">
<i class="fa fa-arrow-circle-o-right"></i>
</button>
<ul class="dropdown-menu" style="width:250px"
@ -130,7 +130,7 @@ limitations under the License.
ng-hide="viewOnly"
ng-click="checkpointNote(note.checkpoint.message)"
style="margin-left: 4px;"
tooltip-placement="bottom" tooltip="Commit this note">Commit
tooltip-placement="bottom" uib-tooltip="Commit this note">Commit
</button>
</div>
</div>
@ -169,7 +169,7 @@ limitations under the License.
class="btn btn-default btn-xs"
ng-click="removeNote(note.id)"
ng-hide="viewOnly"
tooltip-placement="bottom" tooltip="Remove this note permanently"
tooltip-placement="bottom" uib-tooltip="Remove this note permanently"
ng-disabled="revisionView">
<i class="icon-trash"></i>
</button>
@ -179,7 +179,7 @@ limitations under the License.
class="btn btn-default btn-xs"
ng-click="moveNoteToTrash(note.id)"
ng-hide="viewOnly"
tooltip-placement="bottom" tooltip="Move this note to trash"
tooltip-placement="bottom" uib-tooltip="Move this note to trash"
ng-disabled="revisionView">
<i class="icon-trash"></i>
</button>
@ -191,7 +191,7 @@ limitations under the License.
type="button"
data-toggle="dropdown"
ng-class="{ 'btn-info' : note.config.cron, 'btn-danger' : note.info.cron, 'btn-default' : !note.config.cron}"
tooltip-placement="bottom" tooltip="Run scheduler"
tooltip-placement="bottom" uib-tooltip="Run scheduler"
ng-disabled="revisionView">
<span class="fa fa-clock-o"></span> {{getCronOptionNameFromValue(note.config.cron)}}
</div>
@ -244,19 +244,19 @@ limitations under the License.
type="button"
data-toggle="modal"
data-target="#shortcutModal"
tooltip-placement="bottom" tooltip="List of shortcut">
tooltip-placement="bottom" uib-tooltip="List of shortcut">
<i class="fa fa-keyboard-o"></i>
</span>
<span class="setting-btn"
type="button"
ng-click="toggleSetting()"
tooltip-placement="bottom" tooltip="Interpreter binding">
tooltip-placement="bottom" uib-tooltip="Interpreter binding">
<i class="fa fa-cog" ng-style="{color: showSetting ? '#3071A9' : 'black' }"></i>
</span>
<span class="setting-btn"
type="button"
ng-click="togglePermissions()"
tooltip-placement="bottom" tooltip="Note permissions">
tooltip-placement="bottom" uib-tooltip="Note permissions">
<i class="fa fa-lock" ng-style="{color: showPermissions ? '#3071A9' : 'black' }"></i>
</span>
</span>

2
zeppelin-web/src/app/notebook/notebook.html
View file

@ -35,7 +35,7 @@ limitations under the License.
<div>
<a ng-click="restartInterpreter(item)"
ng-class="{'inactivelink': !item.selected}"
tooltip="Restart">
uib-tooltip="Restart">
<span class="glyphicon glyphicon-refresh btn-md"></span>
</a>&nbsp
<div as-sortable-item-handle

18
zeppelin-web/src/app/notebook/paragraph/paragraph-control.html
View file

@ -16,13 +16,13 @@ limitations under the License.
<span>
<span ng-show="paragraph.runtimeInfos.jobUrl.values.length == 1">
<a href="{{paragraph.runtimeInfos.jobUrl.values[0]}}" target="_blank" style="text-decoration: none;"
tooltip-placement="top" tooltip="{{paragraph.runtimeInfos.jobUrl.tooltip}}" >
tooltip-placement="top" uib-tooltip="{{paragraph.runtimeInfos.jobUrl.tooltip}}" >
<span class="fa fa-tasks"></span>
{{paragraph.runtimeInfos.jobUrl.label}}
</a>
</span>
<span class="dropdown" ng-show="paragraph.runtimeInfos.jobUrl.values.length > 1">
<span style="cursor:pointer;color:#3071A9" tooltip-placement="top" tooltip="{{paragraph.runtimeInfos.jobUrl.tooltip}}"
<span style="cursor:pointer;color:#3071A9" tooltip-placement="top" uib-tooltip="{{paragraph.runtimeInfos.jobUrl.tooltip}}"
data-toggle="dropdown" type="button">
<span class="fa fa-tasks"></span>
{{paragraph.runtimeInfos.jobUrl.label}}S
@ -44,18 +44,18 @@ limitations under the License.
<!-- Run / Cancel button -->
<span ng-if="!revisionView">
<span class="icon-control-play" style="cursor:pointer;color:#3071A9" tooltip-placement="top" tooltip="Run this paragraph (Shift+Enter)"
<span class="icon-control-play" style="cursor:pointer;color:#3071A9" tooltip-placement="top" uib-tooltip="Run this paragraph (Shift+Enter)"
ng-click="runParagraphFromButton(getEditorValue())"
ng-show="paragraph.status!='RUNNING' && paragraph.status!='PENDING' && paragraph.config.enabled"></span>
<span class="icon-control-pause" style="cursor:pointer;color:#CD5C5C" tooltip-placement="top"
tooltip="Cancel (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+C)"
uib-tooltip="Cancel (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+C)"
ng-click="cancelParagraph(paragraph)"
ng-show="paragraph.status=='RUNNING' || paragraph.status=='PENDING'"></span>
<span ng-show="paragraph.runtimeInfos.jobUrl.length == 1">
<a href="{{paragraph.runtimeInfos.jobUrl[0]}}" target="_blank"><span class="fa fa-tasks"></span> Spark job </a>
</span>
<span class="dropdown" ng-show="paragraph.runtimeInfos.jobUrl.length > 1">
<span class="fa fa-tasks" style="cursor:pointer;color:#3071A9" tooltip-placement="top" tooltip="Run this paragraph (Shift+Enter)"
<span class="fa fa-tasks" style="cursor:pointer;color:#3071A9" tooltip-placement="top" uib-tooltip="Run this paragraph (Shift+Enter)"
data-toggle="dropdown"
type="button"> Spark Jobs
</span>
@ -66,10 +66,10 @@ limitations under the License.
</ul>
</span>
<span class="{{paragraph.config.editorHide ? 'icon-size-fullscreen' : 'icon-size-actual'}}" style="cursor:pointer" tooltip-placement="top"
tooltip="{{(paragraph.config.editorHide ? 'Show' : 'Hide')}} editor (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+E)"
uib-tooltip="{{(paragraph.config.editorHide ? 'Show' : 'Hide')}} editor (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+E)"
ng-click="toggleEditor(paragraph)"></span>
<span class="{{paragraph.config.tableHide ? 'icon-notebook' : 'icon-book-open'}}" style="cursor:pointer" tooltip-placement="top"
tooltip="{{(paragraph.config.tableHide ? 'Show' : 'Hide')}} output (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+O)"
uib-tooltip="{{(paragraph.config.tableHide ? 'Show' : 'Hide')}} output (Ctrl+{{ (isMac ? 'Option' : 'Alt') }}+O)"
ng-click="toggleOutput(paragraph)"></span>
<span class="dropdown navbar-right">
<span class="icon-settings" style="cursor:pointer"
@ -84,7 +84,7 @@ limitations under the License.
ngclipboard-error="clipError($event)"
data-clipboard-text="{{paragraph.id}}"
tooltip-placement="top"
tooltip="{{tooltip}}">
uib-tooltip="{{tooltip}}">
<span>{{paragraph.id}}</span>
</a>
</li>
@ -96,7 +96,7 @@ limitations under the License.
<form style="display:inline; float:right">
<input type="checkbox"
style="width:16px; margin-right:20px"
tooltip-placement="top" tooltip="Even if you uncheck this, still can run the paragraph by pressing Enter"
tooltip-placement="top" uib-tooltip="Even if you uncheck this, still can run the paragraph by pressing Enter"
ng-checked="{{paragraph.config.runOnSelectionChange}}"
ng-click="turnOnAutoRun(paragraph)"/>
</form>

3
zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html
View file

@ -21,6 +21,7 @@ limitations under the License.
ng-repeat="viz in builtInTableDataVisualizationList track by $index"
ng-class="{'active' : viz.id == graphMode && !config.helium.activeApp}"
ng-click="switchViz(viz.id)"
tooltip-placement="bottom" uib-tooltip="{{viz.name ? viz.name : ''}}"
ng-bind-html="viz.icon">
</button>
</div>
@ -77,7 +78,7 @@ limitations under the License.
<button type="button" class="btn btn-default btn-sm"
style="margin-left:10px"
ng-click="exportToDSV(',')"
tooltip="Download Data as CSV" tooltip-placement="bottom">
uib-tooltip="Download Data as CSV" tooltip-placement="bottom">
<i class="fa fa-download"></i>
</button>
<button type="button" class="btn btn-default btn-sm dropdown-toggle caretBtn"

4
zeppelin-web/src/app/notebook/paragraph/result/result.html
View file

@ -56,14 +56,14 @@ limitations under the License.
<div class="fa fa-level-down scroll-paragraph-down"
ng-show="showScrollDownIcon()"
ng-click="scrollParagraphDown()"
tooltip="Follow Output"></div>
uib-tooltip="Follow Output"></div>
<div id="p{{id}}_text"
style="max-height: {{config.graph.height}}px; overflow: auto"
class="text plainTextContent"></div>
<div class="fa fa-chevron-up scroll-paragraph-up"
ng-show="showScrollUpIcon()"
ng-click="scrollParagraphUp()"
tooltip="Scroll Top"></div>
uib-tooltip="Scroll Top"></div>
</div>
<div id="p{{id}}_custom" class="resultContained"

100
zeppelin-web/src/components/helium/helium.service.js
View file

@ -27,41 +27,14 @@ angular.module('zeppelinWebApp').service('heliumService', heliumService);
export default function heliumService($http, $sce, baseUrlSrv) {
'ngInject';
var url = baseUrlSrv.getRestApiBase() + '/helium/bundle/load';
if (process.env.HELIUM_BUNDLE_DEV) {
url = url + '?refresh=true';
}
let visualizationBundles = [];
// name `heliumBundles` should be same as `HelumBundleFactory.HELIUM_BUNDLES_VAR`
// name `heliumBundles` should be same as `HeliumBundleFactory.HELIUM_BUNDLES_VAR`
let heliumBundles = [];
// map for `{ magic: interpreter }`
let spellPerMagic = {};
// map for `{ magic: package-name }`
let pkgNamePerMagic = {}
// load should be promise
this.load = $http.get(url).success(function(response) {
if (response.substring(0, 'ERROR:'.length) !== 'ERROR:') {
// evaluate bundles
eval(response);
// extract bundles by type
heliumBundles.map(b => {
if (b.type === HeliumType.SPELL) {
const spell = new b.class(); // eslint-disable-line new-cap
const pkgName = b.id;
spellPerMagic[spell.getMagic()] = spell;
pkgNamePerMagic[spell.getMagic()] = pkgName;
} else if (b.type === HeliumType.VISUALIZATION) {
visualizationBundles.push(b);
}
});
} else {
console.error(response);
}
});
/**
* @param magic {string} e.g `%flowchart`
* @returns {SpellBase} undefined if magic is not registered
@ -160,6 +133,37 @@ export default function heliumService($http, $sce, baseUrlSrv) {
});
};
this.getAllEnabledPackages = function() {
return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/enabledPackage`)
.then(function(response, status) {
return response.data.body;
})
.catch(function(error) {
console.error('Failed to get all enabled package infos', error);
});
};
this.getSingleBundle = function(pkgName) {
let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}`
if (process.env.HELIUM_BUNDLE_DEV) {
url = url + '?refresh=true';
}
return $http.get(url)
.then(function(response, status) {
const bundle = response.data
if (bundle.substring(0, 'ERROR:'.length) === 'ERROR:') {
console.error(`Failed to get bundle: ${pkgName}`, bundle);
return '' // empty bundle will be filtered later
}
return bundle
})
.catch(function(error) {
console.error(`Failed to get single bundle: ${pkgName}`, error);
});
}
this.getDefaultPackages = function() {
return this.getAllPackageInfo()
.then(pkgSearchResults => {
@ -241,4 +245,44 @@ export default function heliumService($http, $sce, baseUrlSrv) {
return merged;
});
}
const p = this.getAllEnabledPackages()
.then(enabledPackageSearchResults => {
const promises = enabledPackageSearchResults.map(packageSearchResult => {
const pkgName = packageSearchResult.pkg.name
return this.getSingleBundle(pkgName)
})
return Promise.all(promises)
})
.then(bundles => {
return bundles.reduce((acc, b) => {
// filter out empty bundle
if (b === '') { return acc }
acc.push(b)
return acc;
}, [])
})
// load should be promise
this.load = p.then(availableBundles => {
// evaluate bundles
availableBundles.map(b => {
eval(b)
})
// extract bundles by type
heliumBundles.map(b => {
if (b.type === HeliumType.SPELL) {
const spell = new b.class() // eslint-disable-line new-cap
const pkgName = b.id
spellPerMagic[spell.getMagic()] = spell
pkgNamePerMagic[spell.getMagic()] = pkgName
} else if (b.type === HeliumType.VISUALIZATION) {
visualizationBundles.push(b)
}
})
})
}

4
zeppelin-web/src/components/navbar/navbar.html
View file

@ -80,9 +80,9 @@ limitations under the License.
<li style="margin-left: 10px;">
<div class="dropdown">
<i ng-if="navbar.connected" class="fa fa-circle server-connected"
tooltip="WebSocket Connected" tooltip-placement="bottom" style="margin-top: 7px; margin-right: 0px; vertical-align: top"></i>
uib-tooltip="WebSocket Connected" tooltip-placement="bottom" style="margin-top: 7px; margin-right: 0px; vertical-align: top"></i>
<i ng-if="!navbar.connected" class="fa fa-circle server-disconnected"
tooltip="WebSocket Disconnected" tooltip-placement="bottom" style="margin-top: 7px; vertical-align: top"></i>
uib-tooltip="WebSocket Disconnected" tooltip-placement="bottom" style="margin-top: 7px; vertical-align: top"></i>
<button ng-if="ticket" class="nav-btn dropdown-toggle" type="button" data-toggle="dropdown" style="margin:11px 5px 0 0; padding-left: 0px;">
<span class="username">{{ticket.principal}}</span>
<span class="caret" style="margin-bottom: 8px"></span>

5
zeppelin-zengine/pom.xml
View file

@ -290,6 +290,11 @@
<artifactId>mongo-java-driver</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.5</version>
</dependency>
</dependencies>

36
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
View file

@ -242,6 +242,22 @@ public class Helium {
}
}
public List<HeliumPackageSearchResult> getAllEnabledPackages() {
Map<String, List<HeliumPackageSearchResult>> allPackages = getAllPackageInfoWithoutRefresh();
List<HeliumPackageSearchResult> enabledPackages = new ArrayList<>();
for (List<HeliumPackageSearchResult> versionedPackages : allPackages.values()) {
for (HeliumPackageSearchResult psr : versionedPackages) {
if (psr.isEnabled()) {
enabledPackages.add(psr);
break;
}
}
}
return enabledPackages;
}
public List<HeliumPackageSearchResult> getSinglePackageInfo(String packageName) {
Map<String, List<HeliumPackageSearchResult>> result = getAllPackageInfo(false, packageName);
@ -281,8 +297,8 @@ public class Helium {
return null;
}
public File recreateBundle() throws IOException {
return bundleFactory.buildBundle(getBundlePackagesToBundle(), true);
public File getBundle(HeliumPackage pkg, boolean rebuild) throws IOException {
return bundleFactory.buildPackage(pkg, rebuild, true);
}
public void enable(String name, String artifact) throws IOException {
@ -293,14 +309,13 @@ public class Helium {
return;
}
// enable package
heliumConf.enablePackage(name, artifact);
// if package is visualization, rebuild bundle
if (HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
bundleFactory.buildBundle(getBundlePackagesToBundle());
bundleFactory.buildPackage(pkgInfo.getPkg(), true, true);
}
// update conf and save
heliumConf.enablePackage(name, artifact);
save();
}
@ -311,13 +326,8 @@ public class Helium {
return;
}
// update conf and save
heliumConf.disablePackage(name);
HeliumPackageSearchResult pkgInfo = getPackageInfo(name, artifact);
if (pkgInfo == null || HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
bundleFactory.buildBundle(getBundlePackagesToBundle());
}
save();
}
@ -445,7 +455,7 @@ public class Helium {
heliumConf.setBundleDisplayOrder(orderedPackageList);
// if package is visualization, rebuild buildBundle
bundleFactory.buildBundle(getBundlePackagesToBundle());
bundleFactory.buildAllPackages(getBundlePackagesToBundle());
save();
}

563
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
View file

@ -17,10 +17,18 @@
package org.apache.zeppelin.helium;
import com.github.eirslett.maven.plugins.frontend.lib.*;
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Appender;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
@ -30,7 +38,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.util.*;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -42,17 +49,25 @@ public class HeliumBundleFactory {
Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
private final String NODE_VERSION = "v6.9.1";
private final String NPM_VERSION = "3.10.8";
private final String YARN_VERSION = "v0.21.3";
public static final String HELIUM_LOCAL_REPO = "helium-bundle";
public static final String HELIUM_BUNDLES_DIR = "bundles";
public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
public static final String HELIUM_BUNDLES_SRC_DIR = "src";
public static final String HELIUM_BUNDLES_SRC = "load.js";
public static final String PACKAGE_JSON = "package.json";
public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
public static final String HELIUM_BUNDLE = "helium.bundle.js";
public static final String HELIUM_BUNDLES_VAR = "heliumBundles";
private final int FETCH_RETRY_COUNT = 2;
private final int FETCH_RETRY_FACTOR_COUNT = 1;
// Milliseconds
private final int FETCH_RETRY_MIN_TIMEOUT = 5000;
private final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds
private final FrontendPluginFactory frontEndPluginFactory;
private final File workingDirectory;
private final File nodeInstallationDirectory;
private final File heliumLocalRepoDirectory;
private final File heliumBundleDirectory;
private final File heliumLocalModuleDirectory;
private ZeppelinConfiguration conf;
private File tabledataModulePath;
private File visualizationModulePath;
@ -61,18 +76,16 @@ public class HeliumBundleFactory {
private Gson gson;
private boolean nodeAndNpmInstalled = false;
String bundleCacheKey = "";
File currentCacheBundle;
ByteArrayOutputStream out = new ByteArrayOutputStream();
public HeliumBundleFactory(
ZeppelinConfiguration conf,
File nodeInstallationDir,
File moduleDownloadPath,
File tabledataModulePath,
File visualizationModulePath,
File spellModulePath) throws TaskRunnerException {
this(conf, moduleDownloadPath);
this(conf, nodeInstallationDir, moduleDownloadPath);
this.tabledataModulePath = tabledataModulePath;
this.visualizationModulePath = visualizationModulePath;
this.spellModulePath = spellModulePath;
@ -80,16 +93,20 @@ public class HeliumBundleFactory {
public HeliumBundleFactory(
ZeppelinConfiguration conf,
File nodeInstallationDir,
File moduleDownloadPath) throws TaskRunnerException {
this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
this.heliumLocalRepoDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
this.conf = conf;
this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
File installDirectory = workingDirectory;
nodeInstallationDirectory = (nodeInstallationDir == null) ?
heliumLocalRepoDirectory : nodeInstallationDir;
frontEndPluginFactory = new FrontendPluginFactory(
workingDirectory, installDirectory);
heliumLocalRepoDirectory, nodeInstallationDirectory);
currentCacheBundle = new File(workingDirectory, HELIUM_BUNDLE_CACHE);
gson = new Gson();
}
@ -98,13 +115,18 @@ public class HeliumBundleFactory {
return;
}
try {
NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
nodeInstaller.setNodeVersion(NODE_VERSION);
nodeInstaller.install();
NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
npmInstaller.setNpmVersion(NPM_VERSION);
npmInstaller.install();
NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
nodeInstaller.setNodeVersion(NODE_VERSION);
nodeInstaller.install();
YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig());
yarnInstaller.setYarnVersion(YARN_VERSION);
yarnInstaller.install();
configureLogger();
nodeAndNpmInstalled = true;
} catch (InstallationException e) {
@ -117,31 +139,290 @@ public class HeliumBundleFactory {
return new ProxyConfig(proxy);
}
public File buildBundle(List<HeliumPackage> pkgs) throws IOException {
return buildBundle(pkgs, false);
public void buildAllPackages(List<HeliumPackage> pkgs) throws IOException {
buildAllPackages(pkgs, false);
}
public synchronized File buildBundle(List<HeliumPackage> pkgs, boolean forceRefresh)
public File getHeliumPackageDirectory(String pkgName) {
return new File(heliumBundleDirectory, pkgName);
}
public File getHeliumPackageSourceDirectory(String pkgName) {
return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLES_SRC_DIR);
}
public File getHeliumPackageBundleCache(String pkgName) {
return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLE_CACHE);
}
public static List<String> unTgz(File tarFile, File directory) throws IOException {
List<String> result = new ArrayList<String>();
InputStream is = new FileInputStream(tarFile);
GzipCompressorInputStream gcis = new GzipCompressorInputStream(is);
TarArchiveInputStream in = new TarArchiveInputStream(gcis);
TarArchiveEntry entry = in.getNextTarEntry();
while (entry != null) {
if (entry.isDirectory()) {
entry = in.getNextTarEntry();
continue;
}
File curfile = new File(directory, entry.getName());
File parent = curfile.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
OutputStream out = new FileOutputStream(curfile);
IOUtils.copy(in, out);
out.close();
result.add(entry.getName());
entry = in.getNextTarEntry();
}
in.close();
return result;
}
/**
* @return main file name of this helium package (relative path)
*/
public String downloadPackage(HeliumPackage pkg, String[] nameAndVersion, File bundleDir,
String templateWebpackConfig, String templatePackageJson,
FrontendPluginFactory fpf) throws IOException, TaskRunnerException {
if (bundleDir.exists()) {
FileUtils.deleteQuietly(bundleDir);
}
FileUtils.forceMkdir(bundleDir);
FileFilter copyFilter = new FileFilter() {
@Override
public boolean accept(File pathname) {
String fileName = pathname.getName();
if (fileName.startsWith(".") || fileName.startsWith("#") || fileName.startsWith("~")) {
return false;
} else {
return true;
}
}
};
if (isLocalPackage(pkg)) {
FileUtils.copyDirectory(
new File(pkg.getArtifact()),
bundleDir,
copyFilter);
} else {
// if online package
String version = nameAndVersion[1];
File tgz = new File(heliumLocalRepoDirectory, pkg.getName() + "-" + version + ".tgz");
tgz.delete();
// wget, extract and move dir to `bundles/${pkg.getName()}`, and remove tgz
npmCommand(fpf, "pack " + pkg.getArtifact());
File extracted = new File(heliumBundleDirectory, "package");
FileUtils.deleteDirectory(extracted);
unTgz(tgz, heliumBundleDirectory);
tgz.delete();
FileUtils.copyDirectory(extracted, bundleDir);
FileUtils.deleteDirectory(extracted);
}
// 1. setup package.json
File existingPackageJson = new File(bundleDir, "package.json");
JsonReader reader = new JsonReader(new FileReader(existingPackageJson));
Map<String, Object> packageJson = gson.fromJson(reader,
new TypeToken<Map<String, Object>>(){}.getType());
Map<String, String> existingDeps = (Map<String, String>) packageJson.get("dependencies");
String mainFileName = (String) packageJson.get("main");
StringBuilder dependencies = new StringBuilder();
int index = 0;
for (Map.Entry<String, String> e: existingDeps.entrySet()) {
dependencies.append(" \"").append(e.getKey()).append("\": ");
if (e.getKey().equals("zeppelin-vis") ||
e.getKey().equals("zeppelin-tabledata") ||
e.getKey().equals("zeppelin-spell")) {
dependencies.append("\"file:../../" + HELIUM_LOCAL_MODULE_DIR + "/")
.append(e.getKey()).append("\"");
} else {
dependencies.append("\"").append(e.getValue()).append("\"");
}
if (index < existingDeps.size() - 1) {
dependencies.append(",\n");
}
index = index + 1;
}
FileUtils.deleteQuietly(new File(bundleDir, PACKAGE_JSON));
templatePackageJson = templatePackageJson.replaceFirst("PACKAGE_NAME", pkg.getName());
templatePackageJson = templatePackageJson.replaceFirst("MAIN_FILE", mainFileName);
templatePackageJson = templatePackageJson.replaceFirst("DEPENDENCIES", dependencies.toString());
FileUtils.write(new File(bundleDir, PACKAGE_JSON), templatePackageJson);
// 2. setup webpack.config
FileUtils.write(new File(bundleDir, "webpack.config.js"), templateWebpackConfig);
return mainFileName;
}
public void prepareSource(HeliumPackage pkg, String[] moduleNameVersion,
String mainFileName) throws IOException {
StringBuilder loadJsImport = new StringBuilder();
StringBuilder loadJsRegister = new StringBuilder();
String className = "bundles" + pkg.getName().replaceAll("[-_]", "");
// remove postfix `.js` for ES6 import
if (mainFileName.endsWith(".js")) {
mainFileName = mainFileName.substring(0, mainFileName.length() - 3);
}
loadJsImport
.append("import ")
.append(className)
.append(" from \"../" + mainFileName + "\"\n");
loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
loadJsRegister.append("class: " + className + "\n");
loadJsRegister.append("})\n");
File srcDir = getHeliumPackageSourceDirectory(pkg.getName());
FileUtils.forceMkdir(srcDir);
FileUtils.write(new File(srcDir, HELIUM_BUNDLES_SRC),
loadJsImport.append(loadJsRegister).toString());
}
public synchronized void installNodeModules(FrontendPluginFactory fpf) throws IOException {
try {
out.reset();
String commandForNpmInstall =
String.format("install --fetch-retries=%d --fetch-retry-factor=%d " +
"--fetch-retry-mintimeout=%d",
FETCH_RETRY_COUNT, FETCH_RETRY_FACTOR_COUNT, FETCH_RETRY_MIN_TIMEOUT);
logger.info("Installing required node modules");
yarnCommand(fpf, commandForNpmInstall);
logger.info("Installed required node modules");
} catch (TaskRunnerException e) {
throw new IOException(e);
}
}
public synchronized File bundleHeliumPackage(FrontendPluginFactory fpf,
File bundleDir) throws IOException {
try {
out.reset();
logger.info("Bundling helium packages");
yarnCommand(fpf, "run bundle");
logger.info("Bundled helium packages");
} catch (TaskRunnerException e) {
throw new IOException(new String(out.toByteArray()));
}
String bundleStdoutResult = new String(out.toByteArray());
File heliumBundle = new File(bundleDir, HELIUM_BUNDLE);
if (!heliumBundle.isFile()) {
throw new IOException(
"Can't create bundle: \n" + bundleStdoutResult);
}
WebpackResult result = getWebpackResultFromOutput(bundleStdoutResult);
if (result.errors.length > 0) {
FileUtils.deleteQuietly(heliumBundle);
throw new IOException(result.errors[0]);
}
return heliumBundle;
}
public synchronized File buildPackage(HeliumPackage pkg,
boolean rebuild,
boolean recopyLocalModule) throws IOException {
if (pkg == null) {
return null;
}
String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
if (moduleNameVersion == null) {
return null;
}
if (moduleNameVersion == null) {
logger.error("Can't get module name and version of package " + pkg.getName());
return null;
}
String pkgName = pkg.getName();
File bundleDir = getHeliumPackageDirectory(pkgName);
File bundleCache = getHeliumPackageBundleCache(pkgName);
if (!rebuild && bundleCache.exists() && !bundleCache.isDirectory()) {
return bundleCache;
}
// 0. install node, npm (should be called before `downloadPackage`
installNodeAndNpm();
// 1. prepare directories
if (!heliumLocalRepoDirectory.exists() || !heliumLocalRepoDirectory.isDirectory()) {
FileUtils.deleteQuietly(heliumLocalRepoDirectory);
FileUtils.forceMkdir(heliumLocalRepoDirectory);
}
FrontendPluginFactory fpf = new FrontendPluginFactory(
bundleDir, nodeInstallationDirectory);
// resources: webpack.js, package.json
String templateWebpackConfig = Resources.toString(
Resources.getResource("helium/webpack.config.js"), Charsets.UTF_8);
String templatePackageJson = Resources.toString(
Resources.getResource("helium/" + PACKAGE_JSON), Charsets.UTF_8);
// 2. download helium package using `npm pack`
String mainFileName = null;
try {
mainFileName = downloadPackage(pkg, moduleNameVersion, bundleDir,
templateWebpackConfig, templatePackageJson, fpf);
} catch (TaskRunnerException e) {
throw new IOException(e);
}
// 3. prepare bundle source
prepareSource(pkg, moduleNameVersion, mainFileName);
// 4. install node and local modules for a bundle
copyFrameworkModuleToInstallPath(recopyLocalModule); // should copy local modules first
installNodeModules(fpf);
// 5. let's bundle and update cache
File heliumBundle = bundleHeliumPackage(fpf, bundleDir);
bundleCache.delete();
FileUtils.moveFile(heliumBundle, bundleCache);
return bundleCache;
}
public synchronized void buildAllPackages(List<HeliumPackage> pkgs, boolean rebuild)
throws IOException {
if (pkgs == null || pkgs.size() == 0) {
// when no package is selected, simply return an empty file instead of try bundle package
synchronized (this) {
currentCacheBundle.getParentFile().mkdirs();
currentCacheBundle.delete();
currentCacheBundle.createNewFile();
bundleCacheKey = "";
return currentCacheBundle;
}
return;
}
installNodeAndNpm();
// DON't recopy local modules when build all packages to avoid duplicated copies.
boolean recopyLocalModules = false;
// package.json
URL pkgUrl = Resources.getResource("helium/package.json");
String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8);
StringBuilder dependencies = new StringBuilder();
StringBuilder cacheKeyBuilder = new StringBuilder();
for (HeliumPackage pkg : pkgs) {
try {
buildPackage(pkg, rebuild, recopyLocalModules);
} catch (IOException e) {
logger.error("Failed to build helium package: " + pkg.getArtifact(), e);
}
}
}
void copyFrameworkModuleToInstallPath(boolean recopy)
throws IOException {
FileFilter npmPackageCopyFilter = new FileFilter() {
@Override
@ -155,186 +436,79 @@ public class HeliumBundleFactory {
}
};
for (HeliumPackage pkg : pkgs) {
String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
if (moduleNameVersion == null) {
logger.error("Can't get module name and version of package " + pkg.getName());
continue;
}
if (dependencies.length() > 0) {
dependencies.append(",\n");
}
dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\"");
cacheKeyBuilder.append(pkg.getName() + pkg.getArtifact());
// install tabledata module
FileUtils.forceMkdir(heliumLocalModuleDirectory);
File tabledataModuleInstallPath = new File(heliumLocalModuleDirectory,
"zeppelin-tabledata");
if (tabledataModulePath != null) {
if (recopy && tabledataModuleInstallPath.exists()) {
FileUtils.deleteDirectory(tabledataModuleInstallPath);
File pkgInstallDir = new File(workingDirectory, "node_modules/" + pkg.getName());
if (pkgInstallDir.exists()) {
FileUtils.deleteDirectory(pkgInstallDir);
}
if (isLocalPackage(pkg)) {
if (!tabledataModuleInstallPath.exists()) {
FileUtils.copyDirectory(
new File(pkg.getArtifact()),
pkgInstallDir,
tabledataModulePath,
tabledataModuleInstallPath,
npmPackageCopyFilter);
}
}
pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString());
// check if we can use previous buildBundle or not
if (cacheKeyBuilder.toString().equals(bundleCacheKey) &&
currentCacheBundle.isFile() && !forceRefresh) {
return currentCacheBundle;
}
// webpack.config.js
URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js");
String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8);
// generate load.js
StringBuilder loadJsImport = new StringBuilder();
StringBuilder loadJsRegister = new StringBuilder();
long idx = 0;
for (HeliumPackage pkg : pkgs) {
String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
if (moduleNameVersion == null) {
continue;
}
String className = "bundles" + idx++;
loadJsImport.append(
"import " + className + " from \"" + moduleNameVersion[0] + "\"\n");
loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
loadJsRegister.append("class: " + className + "\n");
loadJsRegister.append("})\n");
}
FileUtils.write(new File(workingDirectory, "package.json"), pkgJson);
FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig);
FileUtils.write(new File(workingDirectory, "load.js"),
loadJsImport.append(loadJsRegister).toString());
copyFrameworkModuleToInstallPath(npmPackageCopyFilter);
try {
out.reset();
String commandForNpmInstall =
String.format("install --fetch-retries=%d --fetch-retry-factor=%d " +
"--fetch-retry-mintimeout=%d",
FETCH_RETRY_COUNT, FETCH_RETRY_FACTOR_COUNT, FETCH_RETRY_MIN_TIMEOUT);
logger.info("Installing required node modules");
npmCommand(commandForNpmInstall);
logger.info("Installed required node modules");
} catch (TaskRunnerException e) {
// ignore `(empty)` warning
String cause = new String(out.toByteArray());
if (!cause.contains("(empty)")) {
throw new IOException(cause);
}
}
try {
out.reset();
logger.info("Bundling helium packages");
npmCommand("run bundle");
logger.info("Bundled helium packages");
} catch (TaskRunnerException e) {
throw new IOException(new String(out.toByteArray()));
}
String bundleStdoutResult = new String(out.toByteArray());
File heliumBundle = new File(workingDirectory, HELIUM_BUNDLE);
if (!heliumBundle.isFile()) {
throw new IOException(
"Can't create bundle: \n" + bundleStdoutResult);
}
WebpackResult result = getWebpackResultFromOutput(bundleStdoutResult);
if (result.errors.length > 0) {
heliumBundle.delete();
throw new IOException(result.errors[0]);
}
synchronized (this) {
currentCacheBundle.delete();
FileUtils.moveFile(heliumBundle, currentCacheBundle);
bundleCacheKey = cacheKeyBuilder.toString();
}
return currentCacheBundle;
}
private void copyFrameworkModuleToInstallPath(FileFilter npmPackageCopyFilter)
throws IOException {
// install tabledata module
File tabledataModuleInstallPath = new File(workingDirectory,
"node_modules/zeppelin-tabledata");
if (tabledataModulePath != null) {
if (tabledataModuleInstallPath.exists()) {
FileUtils.deleteDirectory(tabledataModuleInstallPath);
}
FileUtils.copyDirectory(
tabledataModulePath,
tabledataModuleInstallPath,
npmPackageCopyFilter);
}
// install visualization module
File visModuleInstallPath = new File(workingDirectory,
"node_modules/zeppelin-vis");
File visModuleInstallPath = new File(heliumLocalModuleDirectory,
"zeppelin-vis");
if (visualizationModulePath != null) {
if (visModuleInstallPath.exists()) {
// when zeppelin-vis and zeppelin-table package is published to npm repository
// we don't need to remove module because npm install command will take care
// dependency version change. However, when two dependencies are copied manually
// into node_modules directory, changing vis package version results inconsistent npm
// install behavior.
//
// Remote vis package everytime and let npm download every time bundle as a workaround
if (recopy && visModuleInstallPath.exists()) {
FileUtils.deleteDirectory(visModuleInstallPath);
}
FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath, npmPackageCopyFilter);
if (!visModuleInstallPath.exists()) {
FileUtils.copyDirectory(
visualizationModulePath,
visModuleInstallPath,
npmPackageCopyFilter);
}
}
// install spell module
File spellModuleInstallPath = new File(workingDirectory,
"node_modules/zeppelin-spell");
File spellModuleInstallPath = new File(heliumLocalModuleDirectory,
"zeppelin-spell");
if (spellModulePath != null) {
if (spellModuleInstallPath.exists()) {
if (recopy && spellModuleInstallPath.exists()) {
FileUtils.deleteDirectory(spellModuleInstallPath);
}
FileUtils.copyDirectory(
spellModulePath,
spellModuleInstallPath,
npmPackageCopyFilter);
if (!spellModuleInstallPath.exists()) {
FileUtils.copyDirectory(
spellModulePath,
spellModuleInstallPath,
npmPackageCopyFilter);
}
}
}
private WebpackResult getWebpackResultFromOutput(String output) {
BufferedReader reader = new BufferedReader(new StringReader(output));
String line;
boolean webpackRunDetected = false;
boolean resultJsonDetected = false;
StringBuffer sb = new StringBuffer();
try {
while ((line = reader.readLine()) != null) {
String next, line = reader.readLine();
for (boolean last = (line == null); !last; line = next) {
last = ((next = reader.readLine()) == null);
if (!webpackRunDetected) {
if (line.contains("webpack.js") && line.endsWith("--json")) {
String trimed = line.trim();
if (trimed.contains("webpack") && trimed.endsWith("--json")) {
webpackRunDetected = true;
}
continue;
}
if (!resultJsonDetected) {
if (line.equals("{")) {
if (line.trim().equals("{")) {
sb.append(line);
resultJsonDetected = true;
}
@ -342,10 +516,12 @@ public class HeliumBundleFactory {
}
if (resultJsonDetected && webpackRunDetected) {
sb.append(line);
// yarn command always ends with `Done in ... seconds `
if (!last) {
sb.append(line);
}
}
}
Gson gson = new Gson();
return gson.fromJson(sb.toString(), WebpackResult.class);
} catch (IOException e) {
@ -354,16 +530,6 @@ public class HeliumBundleFactory {
}
}
public File getCurrentCacheBundle() {
synchronized (this) {
if (currentCacheBundle.isFile()) {
return currentCacheBundle;
} else {
return null;
}
}
}
private boolean isLocalPackage(HeliumPackage pkg) {
return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/"));
}
@ -423,14 +589,27 @@ public class HeliumBundleFactory {
}
private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
installNodeAndNpm();
NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
npm.execute(args, env);
}
private void configureLogger() {
private void npmCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
npmCommand(args, new HashMap<String, String>());
}
private void yarnCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
yarnCommand(fpf, args, new HashMap<String, String>());
}
private void yarnCommand(FrontendPluginFactory fpf,
String args, Map<String, String> env) throws TaskRunnerException {
YarnRunner yarn = fpf.getYarnRunner(getProxyConfig(), defaultNpmRegistryUrl);
yarn.execute(args, env);
}
private synchronized void configureLogger() {
org.apache.log4j.Logger npmLogger = org.apache.log4j.Logger.getLogger(
"com.github.eirslett.maven.plugins.frontend.lib.DefaultNpmRunner");
"com.github.eirslett.maven.plugins.frontend.lib.DefaultYarnRunner");
Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders();
if (appenders != null) {
@ -440,7 +619,7 @@ public class HeliumBundleFactory {
@Override
public int decide(LoggingEvent loggingEvent) {
if (loggingEvent.getLoggerName().contains("DefaultNpmRunner")) {
if (loggingEvent.getLoggerName().contains("DefaultYarnRunner")) {
return DENY;
} else {
return NEUTRAL;

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
View file

@ -43,7 +43,6 @@ public class HeliumLocalRegistry extends HeliumRegistry {
}
@Override
public synchronized List<HeliumPackage> getAll() throws IOException {
List<HeliumPackage> result = new LinkedList<>();

8
zeppelin-zengine/src/main/resources/helium/package.json
View file

@ -1,11 +1,11 @@
{
"name": "zeppelin-helium-bundle",
"main": "load",
"name": "PACKAGE_NAME",
"main": "MAIN_FILE",
"scripts": {
"bundle": "node/node node_modules/webpack/bin/webpack.js --display-error-details --json"
"bundle": "webpack --display-error-details --json"
},
"dependencies": {
DEPENDENCIES
DEPENDENCIES
},
"devDependencies": {
"webpack": "^1.12.2",

55
zeppelin-zengine/src/main/resources/helium/webpack.config.js
View file

@ -16,43 +16,22 @@
*/
module.exports = {
entry: './load.js',
entry: './src/load.js',
output: { path: './', filename: 'helium.bundle.js', },
module: {
loaders: [
{
test: /\.js$/,
// DON'T exclude. since zeppelin will bundle all necessary packages: `exclude: /node_modules/,`
loader: 'babel-loader',
query: { presets: ['es2015', 'stage-0'] },
},
{
test: /(\.css)$/,
loaders: ['style', 'css?sourceMap&importLoaders=1'],
},
{
test: /\.woff(\?\S*)?$/,
loader: 'url-loader?limit=10000&minetype=application/font-woff',
},
{
test: /\.woff2(\?\S*)?$/,
loader: 'url-loader?limit=10000&minetype=application/font-woff',
},
{
test: /\.eot(\?\S*)?$/,
loader: 'url-loader',
}, {
test: /\.ttf(\?\S*)?$/,
loader: 'url-loader',
},
{
test: /\.svg(\?\S*)?$/,
loader: 'url-loader',
},
{
test: /\.json$/,
loader: 'json-loader'
},
],
}
module: {
loaders: [
{
test: /\.js$/,
loader: 'babel-loader',
exclude: /node_modules\/(?!(zeppelin-spell|zeppelin-vis|zeppelin-tabledata)\/).*/,
query: { presets: ['es2015', 'stage-0'] },
},
{ test: /(\.css)$/, loaders: ['style', 'css?sourceMap&importLoaders=1'], },
{ test: /\.woff(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
{ test: /\.woff2(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
{ test: /\.eot(\?\S*)?$/, loader: 'url-loader', }, {
test: /\.ttf(\?\S*)?$/, loader: 'url-loader', }, {
test: /\.svg(\?\S*)?$/, loader: 'url-loader', }, {
test: /\.json$/, loader: 'json-loader' }, ],
}
}

45
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
View file

@ -22,6 +22,7 @@ import com.google.common.io.Resources;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
@ -38,9 +39,16 @@ public class HeliumBundleFactoryTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private HeliumBundleFactory hbf;
static File nodeInstallationDir = new File(
System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_nodeCache");
@BeforeClass
static public void beforeAll() throws IOException {
FileUtils.deleteDirectory(nodeInstallationDir);
}
@Before
public void setUp() throws InstallationException, TaskRunnerException {
public void setUp() throws InstallationException, TaskRunnerException, IOException {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
tmpDir.mkdirs();
@ -52,10 +60,13 @@ public class HeliumBundleFactoryTest {
conf = new ZeppelinConfiguration();
hbf = new HeliumBundleFactory(conf,
nodeInstallationDir,
tmpDir,
new File(moduleDir, "tabledata"),
new File(moduleDir, "visualization"),
new File(moduleDir, "spell"));
hbf.installNodeAndNpm();
hbf.copyFrameworkModuleToInstallPath(true);
}
@After
@ -65,17 +76,9 @@ public class HeliumBundleFactoryTest {
@Test
public void testInstallNpm() throws InstallationException {
assertFalse(new File(tmpDir,
HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
assertFalse(new File(tmpDir,
HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
hbf.installNodeAndNpm();
assertTrue(new File(tmpDir,
HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
assertTrue(new File(tmpDir,
HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
assertTrue(new File(nodeInstallationDir, "/node/npm").isFile());
assertTrue(new File(nodeInstallationDir, "/node/node").isFile());
assertTrue(new File(nodeInstallationDir, "/node/yarn/dist/bin/yarn").isFile());
}
@Test
@ -107,14 +110,12 @@ public class HeliumBundleFactoryTest {
"license",
"icon"
);
List<HeliumPackage> pkgs = new LinkedList<>();
pkgs.add(pkg);
File bundle = hbf.buildBundle(pkgs);
File bundle = hbf.buildPackage(pkg, true, true);
assertTrue(bundle.isFile());
long lastModified = bundle.lastModified();
// buildBundle again and check if it served from cache
bundle = hbf.buildBundle(pkgs);
bundle = hbf.buildPackage(pkg, false, true);
assertEquals(lastModified, bundle.lastModified());
}
@ -135,9 +136,7 @@ public class HeliumBundleFactoryTest {
"license",
"fa fa-coffee"
);
List<HeliumPackage> pkgs = new LinkedList<>();
pkgs.add(pkg);
File bundle = hbf.buildBundle(pkgs);
File bundle = hbf.buildPackage(pkg, true, true);
assertTrue(bundle.isFile());
}
@ -157,11 +156,9 @@ public class HeliumBundleFactoryTest {
"license",
"fa fa-coffee"
);
List<HeliumPackage> pkgs = new LinkedList<>();
pkgs.add(pkg);
File bundle = null;
try {
bundle = hbf.buildBundle(pkgs);
bundle = hbf.buildPackage(pkg, true, true);
// should throw exception
assertTrue(false);
} catch (IOException e) {
@ -202,8 +199,8 @@ public class HeliumBundleFactoryTest {
List<HeliumPackage> pkgsV2 = new LinkedList<>();
pkgsV2.add(pkgV2);
File bundle1 = hbf.buildBundle(pkgsV1);
File bundle2 = hbf.buildBundle(pkgsV2);
File bundle1 = hbf.buildPackage(pkgV1, true, true);
File bundle2 = hbf.buildPackage(pkgV2, true, true);
assertNotSame(bundle1.lastModified(), bundle2.lastModified());
}