Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'apache/master'

Browse source
This commit is contained in:
Nate Sammons 2016-05-26 07:52:31 -07:00
commit cdd3107bd2
40 changed files with 1684 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -25,6 +25,7 @@ conf/keystore
conf/truststore
conf/interpreter.json
conf/notebook-authorization.json
conf/shiro.ini
# other generated files
spark/dependency-reduced-pom.xml
@ -86,6 +87,9 @@ Thumbs.db
target/
**/target/
# Generated by Jekyll
docs/_site/
*~
\#*\#
/.emacs.desktop

7
bin/interpreter.sh
View file

@ -85,10 +85,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit"
SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)"
# This will evantually passes SPARK_APP_JAR to classpath of SparkIMain
ZEPPELIN_CLASSPATH=${SPARK_APP_JAR}
# Need to add the R Interpreter
RZEPPELINPATH="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-zr*.jar)"
ZEPPELIN_CLASSPATH="${ZEPPELIN_CLASSPATH}:${RZEPPELINPATH}"
ZEPPELIN_CLASSPATH+=${SPARK_APP_JAR}
pattern="$SPARK_HOME/python/lib/py4j-*-src.zip"
py4j=($pattern)
@ -133,8 +130,6 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
RZEPPELINPATH="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-zr*.jar)"
ZEPPELIN_CLASSPATH="${ZEPPELIN_CLASSPATH}:${RZEPPELINPATH}"
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
fi
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then

2
conf/zeppelin-site.xml.template
View file

@ -169,7 +169,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

11
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -54,9 +54,10 @@
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
<li><a href="{{BASE_PATH}}/interpreter/R.html">R</a></li>
<li><a href="{{BASE_PATH}}/interpreter/r.html">R</a></li>
<li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Shell</a></li>
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
@ -93,7 +94,8 @@
<li role="separator" class="divider"></li>
<!-- li><span><b>Security</b><span></li -->
<li><a href="{{BASE_PATH}}/security/overview.html">Security Overview</a></li>
<li><a href="{{BASE_PATH}}/security/authentication.html">Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/authentication.html">Authentication for NGINX</a></li>
<li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/interpreter_authorization.html">Interpreter Authorization</a></li>
<li role="separator" class="divider"></li>
@ -101,12 +103,9 @@
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Shiro Security</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/shiroauthentication.html">Shiro Authentication</a></li>
</ul>
</li>
</ul>
</nav><!--/.navbar-collapse -->
</div>
</div>
</div>

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin-login.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 245 KiB

6
docs/assets/themes/zeppelin/js/docs.js
View file

@ -99,9 +99,7 @@ function viewSolution() {
// A script to fix internal hash links because we have an overlapping top bar.
// Based on https://github.com/twitter/bootstrap/issues/193#issuecomment-2281510
function maybeScrollToHash() {
console.log("HERE");
if (window.location.hash && $(window.location.hash).length) {
console.log("HERE2", $(window.location.hash), $(window.location.hash).offset().top);
var newTop = $(window.location.hash).offset().top - 57;
$(window).scrollTop(newTop);
}
@ -117,5 +115,5 @@ $(function() {
// Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers
// will try to do *their* initial scroll after running the onReady handler.
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });
});
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });
});

107
docs/interpreter/livy.md Normal file
View file

@ -0,0 +1,107 @@
---
layout: page
title: "Livy Interpreter"
description: ""
group: manual
---
{% include JB/setup %}
## Livy Interpreter for Apache Zeppelin
Livy is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN.
* Interactive Scala, Python and R shells
* Batch submissions in Scala, Java, Python
* Multi users can share the same server (impersonation support)
* Can be used for submitting jobs from anywhere with REST
* Does not require any code change to your programs
### Requirements
Additional requirements for the Livy interpreter are:
* Spark 1.3 or above.
* Livy server.
### Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
<th>Default</th>
<th>Description</th>
</tr>
<tr>
<td>zeppelin.livy.master</td>
<td>local[*]</td>
<td>Spark master uri. ex) spark://masterhost:7077</td>
</tr>
<tr>
<td>zeppelin.livy.url</td>
<td>http://localhost:8998</td>
<td>URL where livy server is running</td>
</tr>
<tr>
<td>zeppelin.livy.spark.maxResult</td>
<td>1000</td>
<td>Max number of SparkSQL result to display.</td>
</tr>
</table>
## How to use
Basically, you can use
**spark**
```
%livy.spark
sc.version
```
**pyspark**
```
%livy.pyspark
print "1"
```
**sparkR**
```
%livy.sparkr
hello <- function( name ) {
sprintf( "Hello, %s", name );
}
hello("livy")
```
## Impersonation
When Zeppelin server is running with authentication enabled, then this interpreter utilizes Livy’s user impersonation feature i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server.
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features.
```
%livy.pyspark
print "${group_by=product_id,product_id|product_name|customer_id|store_id}"
```
## FAQ
Livy debugging: If you see any of these in error console
> Connect to livyhost:8998 [livyhost/127.0.0.1, livyhost/0:0:0:0:0:0:0:1] failed: Connection refused
Looks like the livy server is not up yet or the config is wrong
> Exception: Session not found, Livy server would have restarted, or lost session.
The session would have timed out, you may need to restart the interpreter.
> Blacklisted configuration values in session config: spark.master
edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line.

10
docs/manual/interpreters.md
View file

@ -29,18 +29,18 @@ Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_create.png">
## What is Zeppelin Interpreter Setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
Each notebook can be bound to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
@ -51,7 +51,7 @@ By default, every interpreter is belonged to a single group, but the group might
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
## Interpreter binding mode
@ -59,4 +59,4 @@ Each interpreters is belonged to a single group and registered together. All of
Each Interpreter Setting can choose one of 'shared', 'scoped', 'isolated' interpreter binding mode.
In 'shared' mode, every notebook bound to the Interpreter Setting will share the single Interpreter instance. In 'scoped' mode, each notebook will create new Interpreter instance in the same interpreter process. In 'isolated' mode, each notebook will create new Interpreter process.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">

6
docs/security/authentication.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Authentication"
description: "Authentication"
title: "Authentication for NGINX"
description: "Authentication for NGINX"
group: security
---
<!--
@ -17,7 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Authentication
# Authentication for NGINX
Authentication is company-specific.

4
docs/manual/shiroauthentication.md → docs/security/shiroauthentication.md
View file

@ -2,7 +2,7 @@
layout: page
title: "Shiro Security for Apache Zeppelin"
description: ""
group: manual
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -59,7 +59,7 @@ Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
```
admin = password1

55
docs/ui_layout/zeppelin_layout.md
View file

@ -21,21 +21,21 @@ limitations under the License.
The first time you connect to Zeppelin, you'll land at the main page similar to the below screen capture
<img src="/assets/themes/zeppelin/img/ui-img/homepage.png" />
<img src="../assets/themes/zeppelin/img/ui-img/homepage.png" />
On the left of the page are listed all existing notes. Those notes are stored by default in the `$ZEPPELIN_HOME/notebook` folder.
On the left of the page are listed all existing notes. Those notes are stored by default in the `$ZEPPELIN_HOME/notebook` folder.
You can filter them by name using the input text form. You can also create an new note, refresh the list of existing notes
You can filter them by name using the input text form. You can also create an new note, refresh the list of existing notes
(in case you manually copy them into the `$ZEPPELIN_HOME/notebook` folder) and import a note
<img src="/assets/themes/zeppelin/img/ui-img/notes_management.png" />
<img src="../assets/themes/zeppelin/img/ui-img/notes_management.png" />
When clicking on `Import Note` link, a new dialog open. From there you can import your note from local disk or from a remote location
if you provide the URL.
<img src="/assets/themes/zeppelin/img/ui-img/note_import_dialog.png" />
By default, the name of the imported note is the same as the original note but you can override it by providing a new name
<img src="../assets/themes/zeppelin/img/ui-img/note_import_dialog.png" />
By default, the name of the imported note is the same as the original note but you can override it by providing a new name
<br />
## Menus
@ -67,10 +67,10 @@ This menu displays all the Zeppelin configuration that are set in the config fil
<br />
## Note Layout
## Note Layout
Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a paragraph container.
Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a paragraph container.
<img src="../assets/themes/zeppelin/img/ui-img/note_paragraph_layout.png" />
### Paragraph
@ -78,20 +78,20 @@ Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a
Each paragraph consists of 2 sections: `code section` where you put your source code and `result section` where you can see the result of the code execution.
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_layout.png" />
On the top-right corner of each paragraph there are some commands to:
* execute the paragraph code
* hide/show `code section`
* hide/show `result section`
* configure the paragraph
To configure the paragraph, just click on the gear icon:
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_configuration_dialog.png" />
From this dialog, you can (in descending order):
* find the **paragraph id** ( **20150924-163507_134879501** )
* control paragraph width. Since Zeppelin is using the grid system of **Twitter Bootstrap**, each paragraph width can be changed from 1 to 12
* move the paragraph 1 level up
@ -103,19 +103,19 @@ From this dialog, you can (in descending order):
* export the current paragraph as an **iframe** and open the **iframe** in a new window
* clear the `result section`
* delete the current paragraph
### Note toolbar
At the top of the note, you can find a toolbar which exposes command buttons as well as configuration, security and display options
<img src="../assets/themes/zeppelin/img/ui-img/note_toolbar.png" />
<img src="../assets/themes/zeppelin/img/ui-img/note_toolbar.png" />
On the far right is displayed the note name, just click on it to reveal the input form and update it
In the middle of the toolbar you can find the command buttons:
* execute all the paragraphs **sequentially**, in their display order
* hide/show `code section` of all paragraphs
* hide/show `code section` of all paragraphs
* hide/show `result section` of all paragraphs
* clear the `result section` of all paragraphs
* clone the current note
@ -127,13 +127,10 @@ In the middle of the toolbar you can find the command buttons:
<img src="../assets/themes/zeppelin/img/ui-img/note_commands.png" />
On the right of the note tool bar you can find configuration icons:
* display all the keyboard shorcuts
* configure the interpreters binding to the current note
* configure the note permissions
* switch the node display mode between `default`, `simple` and `report`
<img src="../assets/themes/zeppelin/img/ui-img/note_configuration.png" />

8
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -191,13 +191,7 @@ public class JDBCInterpreter extends Interpreter {
logger.info(properties.getProperty(DRIVER_KEY));
Class.forName(properties.getProperty(DRIVER_KEY));
String url = properties.getProperty(URL_KEY);
String user = properties.getProperty(USER_KEY);
String password = properties.getProperty(PASSWORD_KEY);
if (null != user && null != password) {
connection = DriverManager.getConnection(url, user, password);
} else {
connection = DriverManager.getConnection(url, properties);
}
connection = DriverManager.getConnection(url, properties);
}
return connection;
}

170
livy/pom.xml Normal file
View file

@ -0,0 +1,170 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-livy</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Livy interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<!--TEST-->
<junit.version>4.12</junit.version>
<achilles.version>3.2.4-Zeppelin</achilles.version>
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.4</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

409
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java Normal file
View file

@ -0,0 +1,409 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/***
* Livy helper class
*/
public class LivyHelper {
Logger LOGGER = LoggerFactory.getLogger(LivyHelper.class);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
HashMap<String, Object> paragraphHttpMap = new HashMap<>();
Properties property;
Integer MAX_NOS_RETRY = 60;
LivyHelper(Properties property) {
this.property = property;
}
public Integer createSession(InterpreterContext context, String kind) throws Exception {
try {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"master\": \"" + property.getProperty("zeppelin.livy.master") + "\", " +
"\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
"}",
context.getParagraphId()
);
if (json.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"conf\":{\"spark.master\": \""
+ property.getProperty("zeppelin.livy.master") + "\"}," +
"\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
"}",
context.getParagraphId()
);
}
Map jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
Integer sessionId = ((Double) jsonMap.get("id")).intValue();
if (!jsonMap.get("state").equals("idle")) {
Integer nosRetry = MAX_NOS_RETRY;
while (nosRetry >= 0) {
LOGGER.error(String.format("sessionId:%s state is %s",
jsonMap.get("id"), jsonMap.get("state")));
Thread.sleep(1000);
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId,
"GET", null,
context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
if (jsonMap.get("state").equals("idle")) {
break;
} else if (jsonMap.get("state").equals("error")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId + "/log",
"GET", null,
context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
String logs = StringUtils.join((ArrayList<String>) jsonMap.get("log"), '\n');
LOGGER.error(String.format("Cannot start %s.\n%s", kind, logs));
throw new Exception(String.format("Cannot start %s.\n%s", kind, logs));
}
nosRetry--;
}
if (nosRetry <= 0) {
LOGGER.error("Error getting session for user within 60Sec.");
throw new Exception(String.format("Cannot start %s.", kind));
}
}
return sessionId;
} catch (Exception e) {
LOGGER.error("Error getting session for user", e);
throw e;
}
}
protected void initializeSpark(final InterpreterContext context,
final Map<String, Integer> userSessionMap) throws Exception {
interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
"import sqlContext.implicits._", context, userSessionMap);
}
public InterpreterResult interpretInput(String stringLines,
final InterpreterContext context,
final Map<String, Integer> userSessionMap,
LivyOutputStream out) {
try {
String[] lines = stringLines.split("\n");
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
out.setInterpreterOutput(context.out);
context.out.clear();
Code r = null;
String incomplete = "";
boolean inComment = false;
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
//for spark
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
boolean continuation = false;
if (nextLine.isEmpty()
|| nextLine.startsWith("//") // skip empty line or comment
|| nextLine.startsWith("}")
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
continuation = true;
} else if (!inComment && nextLine.startsWith("/*")) {
inComment = true;
continuation = true;
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
inComment = false;
continuation = true;
} else if (nextLine.length() > 1
&& nextLine.charAt(0) == '.'
&& nextLine.charAt(1) != '.' // ".."
&& nextLine.charAt(1) != '/') { // "./"
continuation = true;
} else if (inComment) {
continuation = true;
}
if (continuation) {
incomplete += s + "\n";
continue;
}
}
InterpreterResult res;
try {
res = interpret(incomplete + s, context, userSessionMap);
} catch (Exception e) {
LOGGER.error("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = res.code();
if (r == Code.ERROR) {
out.setInterpreterOutput(null);
return res;
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
out.write((res.message() + "\n").getBytes(Charset.forName("UTF-8")));
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
out.setInterpreterOutput(null);
return new InterpreterResult(r, "Incomplete expression");
} else {
out.setInterpreterOutput(null);
return new InterpreterResult(Code.SUCCESS);
}
} catch (Exception e) {
LOGGER.error("error in interpretInput", e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
public InterpreterResult interpret(String stringLines,
final InterpreterContext context,
final Map<String, Integer> userSessionMap)
throws Exception {
stringLines = stringLines
//for "\n" present in string
.replaceAll("\\\\n", "\\\\\\\\n")
//for new line present in string
.replaceAll("\\n", "\\\\n")
// for \" present in string
.replaceAll("\\\\\"", "\\\\\\\\\"")
// for " present in string
.replaceAll("\"", "\\\\\"");
if (stringLines.trim().equals("")) {
return new InterpreterResult(Code.SUCCESS, "");
}
Map jsonMap = executeCommand(stringLines, context, userSessionMap);
Integer id = ((Double) jsonMap.get("id")).intValue();
InterpreterResult res = getResultFromMap(jsonMap);
if (res != null) {
return res;
}
while (true) {
Thread.sleep(1000);
if (paragraphHttpMap.get(context.getParagraphId()) == null) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
jsonMap = getStatusById(context, userSessionMap, id);
InterpreterResult interpreterResult = getResultFromMap(jsonMap);
if (interpreterResult != null) {
return interpreterResult;
}
}
}
private InterpreterResult getResultFromMap(Map jsonMap) {
if (jsonMap.get("state").equals("available")) {
if (((Map) jsonMap.get("output")).get("status").equals("error")) {
StringBuilder errorMessage = new StringBuilder((String) ((Map) jsonMap
.get("output")).get("evalue"));
if (errorMessage.toString().equals("incomplete statement")
|| errorMessage.toString().contains("EOF")) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
String traceback = gson.toJson(((Map) jsonMap.get("output")).get("traceback"));
if (!traceback.equals("[]")) {
errorMessage
.append("\n")
.append("traceback: \n")
.append(traceback);
}
return new InterpreterResult(Code.ERROR, errorMessage.toString());
}
if (((Map) jsonMap.get("output")).get("status").equals("ok")) {
String result = (String) ((Map) ((Map) jsonMap.get("output"))
.get("data")).get("text/plain");
if (result != null) {
result = result.trim();
if (result.startsWith("<link")
|| result.startsWith("<script")
|| result.startsWith("<style")
|| result.startsWith("<div")) {
result = "%html " + result;
}
}
return new InterpreterResult(Code.SUCCESS, result);
}
}
return null;
}
private Map executeCommand(String lines, InterpreterContext context,
Map<String, Integer> userSessionMap) throws Exception {
String json = executeHTTP(property.get("zeppelin.livy.url") + "/sessions/"
+ userSessionMap.get(context.getAuthenticationInfo().getUser())
+ "/statements",
"POST",
"{\"code\": \"" + lines + "\" }",
context.getParagraphId());
if (json.matches("^(\")?Session (\'[0-9]\' )?not found(.?\"?)$")) {
throw new Exception("Exception: Session not found, Livy server would have restarted, " +
"or lost session.");
}
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error executeCommand", e);
throw e;
}
}
private Map getStatusById(InterpreterContext context,
Map<String, Integer> userSessionMap, Integer id) throws Exception {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
+ userSessionMap.get(context.getAuthenticationInfo().getUser())
+ "/statements/" + id,
"GET", null, context.getParagraphId());
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error getStatusById", e);
throw e;
}
}
protected String executeHTTP(String targetURL, String method, String jsonData, String paragraphId)
throws Exception {
HttpClient client = HttpClientBuilder.create().build();
HttpResponse response = null;
if (method.equals("POST")) {
HttpPost request = new HttpPost(targetURL);
request.addHeader("Content-Type", "application/json");
StringEntity se = new StringEntity(jsonData);
request.setEntity(se);
response = client.execute(request);
paragraphHttpMap.put(paragraphId, request);
} else if (method.equals("GET")) {
HttpGet request = new HttpGet(targetURL);
request.addHeader("Content-Type", "application/json");
response = client.execute(request);
paragraphHttpMap.put(paragraphId, request);
} else if (method.equals("DELETE")) {
HttpDelete request = new HttpDelete(targetURL);
request.addHeader("Content-Type", "application/json");
response = client.execute(request);
}
if (response == null) {
return null;
}
if (response.getStatusLine().getStatusCode() == 200
|| response.getStatusLine().getStatusCode() == 201
|| response.getStatusLine().getStatusCode() == 404) {
return getResponse(response);
} else {
String responseString = getResponse(response);
if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
return responseString;
}
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusLine().getStatusCode(), responseString));
throw new Exception(String.format("Error with %s StatusCode: %s",
response.getStatusLine().getStatusCode(), responseString));
}
}
private String getResponse(HttpResponse response) throws Exception {
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
return result.toString();
}
public void cancelHTTP(String paragraphId) {
if (paragraphHttpMap.get(paragraphId).getClass().getName().contains("HttpPost")) {
((HttpPost) paragraphHttpMap.get(paragraphId)).abort();
} else {
((HttpGet) paragraphHttpMap.get(paragraphId)).abort();
}
paragraphHttpMap.put(paragraphId, null);
}
public void closeSession(Map<String, Integer> userSessionMap) {
for (Map.Entry<String, Integer> entry : userSessionMap.entrySet()) {
try {
executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
+ entry.getValue(),
"DELETE", null, null);
} catch (Exception e) {
LOGGER.error(String.format("Error closing session for user with session ID: %s",
entry.getValue()), e);
}
}
}
}

75
livy/src/main/java/org/apache/zeppelin/livy/LivyOutputStream.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import java.io.IOException;
import java.io.OutputStream;
/**
* InterpreterOutput can be attached / detached.
*/
public class LivyOutputStream extends OutputStream {
InterpreterOutput interpreterOutput;
public LivyOutputStream() {
}
public InterpreterOutput getInterpreterOutput() {
return interpreterOutput;
}
public void setInterpreterOutput(InterpreterOutput interpreterOutput) {
this.interpreterOutput = interpreterOutput;
}
@Override
public void write(int b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte[] b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte[] b, int offset, int len) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b, offset, len);
}
}
@Override
public void close() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.close();
}
}
@Override
public void flush() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.flush();
}
}
}

122
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivyPySparkInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivyPySparkInterpreter.class);
static {
Interpreter.register(
"pyspark",
"livy",
LivyPySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
protected LivyHelper livyHelper;
public LivyPySparkInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"pyspark")
);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivyPySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

135
livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java Normal file
View file

@ -0,0 +1,135 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy Spark interpreter for Zeppelin.
*/
public class LivySparkInterpreter extends Interpreter {
static String DEFAULT_URL = "http://localhost:8998";
static String LOCAL = "local[*]";
Logger LOGGER = LoggerFactory.getLogger(LivySparkInterpreter.class);
private LivyOutputStream out;
static {
Interpreter.register(
"spark",
"livy",
LivySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.")
.add("zeppelin.livy.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
.build()
);
}
protected static Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
out = new LivyOutputStream();
}
protected static Map<String, Integer> getUserSessionMap() {
return userSessionMap;
}
public void setUserSessionMap(Map<String, Integer> userSessionMap) {
this.userSessionMap = userSessionMap;
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"spark")
);
livyHelper.initializeSpark(interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpretInput(line, interpreterContext, userSessionMap, out);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

122
livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkRInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkRInterpreter.class);
static {
Interpreter.register(
"sparkr",
"livy",
LivySparkRInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkRInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"sparkr")
);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkRInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

165
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java Normal file
View file

@ -0,0 +1,165 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
static String DEFAULT_MAX_RESULT = "1000";
static {
Interpreter.register(
"sql",
"livy",
LivySparkSQLInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.spark.maxResult",
DEFAULT_MAX_RESULT,
"Max number of SparkSQL result to display.")
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkSQLInterpreter(Properties property) {
super(property);
livyHelper = new LivyHelper(property);
userSessionMap = LivySparkInterpreter.getUserSessionMap();
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"spark")
);
livyHelper.initializeSpark(interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
InterpreterResult res = livyHelper.interpret("sqlContext.sql(\"" +
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
property.get("zeppelin.livy.spark.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {
StringBuilder resMsg = new StringBuilder();
resMsg.append("%table ");
String[] rows = res.message().split("\n");
String[] headers = rows[1].split("\\|");
for (int head = 1; head < headers.length; head++) {
resMsg.append(headers[head].trim()).append("\t");
}
resMsg.append("\n");
if (rows[3].indexOf("+") == 0) {
} else {
for (int cols = 3; cols < rows.length - 1; cols++) {
String[] col = rows[cols].split("\\|");
for (int data = 1; data < col.length; data++) {
resMsg.append(col[data].trim()).append("\t");
}
resMsg.append("\n");
}
}
if (rows[rows.length - 1].indexOf("only") == 0) {
resMsg.append("<font color=red>" + rows[rows.length - 1] + ".</font>");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
resMsg.toString()
);
} else {
return res;
}
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

114
livy/src/main/test/org/apache/zeppelin/livy/LivyHelperTest.java Normal file
View file

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.GsonBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.hamcrest.CoreMatchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Properties;
import static org.mockito.Mockito.doReturn;
/**
* Created for org.apache.zeppelin.livy on 22/04/16.
*/
@RunWith(MockitoJUnitRunner.class)
public class LivyHelperTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private static LivyPySparkInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext interpreterContext;
@Mock(answer = Answers.CALLS_REAL_METHODS)
private LivyHelper livyHelper;
@Before
public void prepareContext() throws Exception {
interpreter.userSessionMap = new HashMap<>();
interpreter.userSessionMap.put(null, 1);
Properties properties = new Properties();
properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
livyHelper.property = properties;
livyHelper.paragraphHttpMap = new HashMap<>();
livyHelper.gson = new GsonBuilder().setPrettyPrinting().create();
doReturn("{\"id\":1,\"state\":\"idle\",\"kind\":\"spark\",\"proxyUser\":\"null\",\"log\":[]}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{\"kind\": \"spark\", \"proxyUser\": \"null\"}",
null
);
doReturn("{\"id\":1,\"state\":\"available\",\"output\":{\"status\":\"ok\"," +
"\"execution_count\":1,\"data\":{\"text/plain\":\"1\"}}}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions/1/statements",
"POST",
"{\"code\": \"print(1)\" }",
null
);
}
@Test
public void checkCreateSession() {
try {
Integer sessionId = livyHelper.createSession(interpreterContext, "spark");
collector.checkThat("check sessionId", 1, CoreMatchers.equalTo(sessionId));
} catch (Exception e) {
collector.addError(e);
}
}
@Test
public void checkInterpret() {
try {
InterpreterResult result = livyHelper.interpret("print(1)", interpreterContext, interpreter.userSessionMap);
collector.checkThat("check sessionId", InterpreterResult.Code.SUCCESS, CoreMatchers.equalTo(result.code()));
} catch (Exception e) {
collector.addError(e);
}
}
}

86
livy/src/main/test/org/apache/zeppelin/livy/LivyInterpreterTest.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Properties;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doReturn;
@RunWith(MockitoJUnitRunner.class)
public class LivyInterpreterTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
private static LivyPySparkInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext interpreterContext;
@AfterClass
public static void tearDown() {
interpreter.close();
}
@Before
public void prepareContext() throws Exception {
interpreter = new LivyPySparkInterpreter(new Properties());
interpreter.userSessionMap = new HashMap<>();
interpreter.userSessionMap.put(null, 0);
interpreter.livyHelper = Mockito.mock(LivyHelper.class);
interpreter.open();
doReturn(new InterpreterResult(InterpreterResult.Code.SUCCESS)).when(interpreter.livyHelper)
.interpret("print \"x is 1.\"", interpreterContext, interpreter.userSessionMap);
}
@Test
public void checkInitVariables() throws Exception {
collector.checkThat("Check that, if userSessionMap is made: ",
interpreter.userSessionMap, CoreMatchers.notNullValue());
}
@Test
public void checkBasicInterpreter() throws Exception {
String paragraphString = "print \"x is 1.\"";
final InterpreterResult actual = interpreter.interpret(paragraphString, interpreterContext);
collector.checkThat("Check that, result is computed: ",
actual.code(), CoreMatchers.equalTo(InterpreterResult.Code.SUCCESS));
assertThat(actual).isNotNull();
}
}

1
pom.xml
View file

@ -92,6 +92,7 @@
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>livy</module>
<module>hive</module>
<module>hbase</module>
<module>phoenix</module>

12
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -276,15 +276,21 @@ public class SparkInterpreter extends Interpreter {
classServerUri = (String) classServer.invoke(interpreter.intp());
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
// continue instead of: throw new InterpreterException(e);
// Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method
logger.warn(String.format("Spark method classServerUri not available due to: [%s]",
e.getMessage()));
}
}
SparkConf conf =
new SparkConf()
.setMaster(getProperty("master"))
.setAppName(getProperty("spark.app.name"))
.set("spark.repl.class.uri", classServerUri);
.setAppName(getProperty("spark.app.name"));
if (classServerUri != null) {
conf.set("spark.repl.class.uri", classServerUri);
}
if (jars.length > 0) {
conf.setJars(jars);

6
zeppelin-server/pom.xml
View file

@ -184,9 +184,9 @@
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
<scope>test</scope>
<exclusions>
<exclusion>

17
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -60,7 +61,8 @@ public class NotebookServer extends WebSocketServlet implements
NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener,
RemoteInterpreterProcessListener {
private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class);
Gson gson = new Gson();
Gson gson = new GsonBuilder()
.setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
final Map<String, List<NotebookSocket>> noteSocketMap = new HashMap<>();
final Queue<NotebookSocket> connectedSockets = new ConcurrentLinkedQueue<>();
@ -399,14 +401,17 @@ public class NotebookServer extends WebSocketServlet implements
broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
}
void permissionError(NotebookSocket conn, String op, Set<String> current,
Set<String> allowed) throws IOException {
void permissionError(NotebookSocket conn, String op, Set<String> userAndRoles,
Set<String> allowed) throws IOException {
LOG.info("Cannot {}. Connection readers {}. Allowed readers {}",
op, current, allowed);
op, userAndRoles, allowed);
String userName = userAndRoles.iterator().next();
conn.send(serializeMessage(new Message(OP.AUTH_INFO).put("info",
"Insufficient privileges to " + op + " note.\n\n" +
"Insufficient privileges to " + op + " notebook.\n\n" +
"Allowed users or roles: " + allowed.toString() + "\n\n" +
"User belongs to: " + current.toString())));
"But the user " + userName + " belongs to: " + userAndRoles.toString())));
}
private void sendNote(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook,

33
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
View file

@ -17,11 +17,13 @@
package org.apache.zeppelin.integration;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.hamcrest.CoreMatchers;
import org.junit.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
@ -194,19 +196,19 @@ public class ZeppelinIT extends AbstractZeppelinIT {
}
try {
// navigate to interpreter page
WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
WebElement interpreterLink = driver.findElement(By.xpath("//a[contains(.,'Interpreter')]"));
interpreterLink.click();
// add new dependency to spark interpreter
WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
driver.findElement(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
WebElement depArtifact = pollingWait(By.xpath("//input[@ng-model='setting.depArtifact']"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
String artifact = "org.apache.commons:commons-csv:1.1";
depArtifact.sendKeys(artifact);
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.findElement(By.xpath("//div[contains(@class,'box')][contains(.,'%spark')]//form//button[1]")).click();
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
driver.navigate().back();
createNewNote();
@ -230,20 +232,17 @@ public class ZeppelinIT extends AbstractZeppelinIT {
//delete created notebook for cleanup.
deleteTestNotebook(driver);
sleep(1000, true);
sleep(1000, false);
// reset dependency
interpreterLink.click();
sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"));
sleep(5000, true);
driver.findElement(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
WebElement testDepRemoveBtn = pollingWait(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"), MAX_IMPLICIT_WAIT);
testDepRemoveBtn.click();
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.findElement(By.xpath("//div[contains(@class,'box')][contains(.,'%spark')]//form//button[1]")).click();
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
} catch (Exception e) {
handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
}

3
zeppelin-web/.jshintrc
View file

@ -32,6 +32,7 @@
"ace": false,
"d3": false,
"BootstrapDialog": false,
"Handsontable": false
"Handsontable": false,
"moment": false
}
}

6
zeppelin-web/src/app/home/home.controller.js
View file

@ -37,7 +37,7 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
console.log('Error %o %o', status, data.message);
});
};
var initHome = function() {
websocketMsgSrv.getHomeNotebook();
getZeppelinVersion();
@ -76,8 +76,4 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
node.hidden = !node.hidden;
};
$rootScope.noteName = function(note) {
return arrayOrderingSrv.getNoteName(note);
};
});

4
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -29,7 +29,7 @@ limitations under the License.
style="width:180px">
<select class="form-control input-sm" ng-model="newInterpreterSetting.group"
ng-change="newInterpreterGroupChange()">
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'group'" value="{{availableInterpreter.group}}">
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'group'| orderBy: 'group'" value="{{availableInterpreter.group}}">
{{availableInterpreter.group}}
</option>
</select>
@ -69,7 +69,7 @@ limitations under the License.
<span>Interpreter for note</span>
</div>
<br />
<b>Properties</b>
<table class="table table-striped properties">
<tr>

13
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -75,10 +75,19 @@ limitations under the License.
</div>
<div ng-include src="'app/interpreter/interpreter-create/interpreter-create.html'"></div>
<div class="input-group col-lg-4" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters">
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
</div>
</div>
<div class="box width-full home"
ng-repeat="setting in interpreterSettings">
ng-repeat="setting in interpreterSettings | orderBy: 'group' | filter: searchInterpreter">
<div>
<div class="row interpreter">
<div class="col-md-12">
@ -144,7 +153,7 @@ limitations under the License.
<span>Interpreter for note</span>
</div>
<div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
<em>Currently there are no properties and dependencies set for this interpreter</em>
</div>

12
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -971,18 +971,22 @@ angular.module('zeppelinWebApp')
return '';
}
var user = 'anonymous';
var authInfo = pdata.authenticationInfo;
if (authInfo && authInfo.user) {
if (pdata.authenticationInfo !== null && !isEmpty(pdata.authenticationInfo.user)) {
user = pdata.authenticationInfo.user;
}
var dateUpdated = (pdata.dateUpdated === null) ? 'unknown' : pdata.dateUpdated;
var desc = 'Took ' + (timeMs/1000) + ' seconds. Last updated by ' + user + ' at time ' + dateUpdated + '.';
var desc = 'Took ' +
moment.duration(moment(pdata.dateFinished).diff(moment(pdata.dateStarted))).humanize() +
'. Last updated by ' + user + ' at ' + moment(pdata.dateUpdated).format('MMMM DD YYYY, h:mm:ss A') + '.';
if ($scope.isResultOutdated()){
desc += ' (outdated)';
}
return desc;
};
$scope.getElapsedTime = function() {
return 'Started ' + moment($scope.paragraph.dateStarted).fromNow() + '.';
};
$scope.isResultOutdated = function() {
var pdata = $scope.paragraph;
if (pdata.dateUpdated !==undefined && Date.parse(pdata.dateUpdated) > Date.parse(pdata.dateStarted)){

8
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -82,11 +82,13 @@ table.dataTable.table-condensed .sorting_desc:after {
border: 3px solid #DDDDDD;
}
.paragraph .paragraphFooter {
height: 9px;
.paragraph .executionTime {
color: #999;
font-size: 10px;
font-family: 'Roboto', sans-serif;
}
.paragraph .executionTime {
.paragraph .elapsedTime {
color: #999;
font-size: 10px;
font-family: 'Roboto', sans-serif;

6
zeppelin-web/src/app/notebook/paragraph/paragraph.html
View file

@ -68,5 +68,11 @@ limitations under the License.
id="{{paragraph.id}}_executionTime"
class="executionTime" ng-bind-html="getExecutionTime()">
</div>
<div ng-if = "paragraph.status === 'RUNNING'" class = "paragraphFooterElapsed">
<div
id="{{paragraph.id}}_elapsedTime"
class="elapsedTime" ng-bind-html="getElapsedTime()">
</div>
</div>
</div>
</div>

15
zeppelin-web/src/assets/styles/looknfeel/simple.css
View file

@ -55,9 +55,14 @@ body {
.paragraph .paragraphFooter {
visibility: hidden;
height: 0;
position: relative;
top : -13px;
top : -9px;
z-index: 99;
}
.paragraph .paragraphFooterElapsed {
height: 0px;
float: right;
z-index: 99;
}
@ -67,6 +72,12 @@ body {
margin-right: 5px;
}
.paragraph .elapsedTime {
font-size: 8px;
text-align: right;
margin-right: 5px;
}
.paragraph:hover .paragraphFooter {
visibility: visible;
}

19
zeppelin-web/src/components/navbar/navbar.controller.js
View file

@ -46,15 +46,6 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
vm.connected = param;
});
$rootScope.$on('$locationChangeSuccess', function () {
var path = $location.path();
// hacky solution to clear search bar
// TODO(felizbear): figure out how to make ng-click work in navbar
if (path === '/') {
$scope.searchTerm = '';
}
});
$scope.checkUsername = function () {
if ($rootScope.ticket) {
if ($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
@ -91,8 +82,8 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
});
};
$scope.search = function() {
$location.url(/search/ + $scope.searchTerm);
$scope.search = function(searchTerm) {
$location.url(/search/ + searchTerm);
};
function loadNotes() {
@ -103,6 +94,12 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
return ($routeParams.noteId === noteId);
}
$rootScope.noteName = function(note) {
if (!_.isEmpty(note)) {
return arrayOrderingSrv.getNoteName(note);
}
};
vm.loadNotes = loadNotes;
vm.isActive = isActive;

2
zeppelin-web/src/components/navbar/navbar.html
View file

@ -65,7 +65,7 @@ limitations under the License.
<!--TODO(bzz): move to Typeahead https://angular-ui.github.io/bootstrap -->
<form role="search"
style="width: 300px; display: inline-block; margin: 0 10px"
ng-submit="search()">
ng-submit="search(searchTerm)">
<div class="input-group">
<input
type="text"

10
zeppelin-zengine/pom.xml
View file

@ -115,16 +115,16 @@
<version>1.5.2</version>
<exclusions>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -471,6 +471,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.markdown.Markdown,"
+ "org.apache.zeppelin.angular.AngularInterpreter,"
+ "org.apache.zeppelin.shell.ShellInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
+ "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ "org.apache.zeppelin.hive.HiveInterpreter,"
+ "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ "org.apache.zeppelin.file.HDFSFileInterpreter,"

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -143,7 +143,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
int scriptHeadIndex = 0;
for (int i = 0; i < text.length(); i++) {
char ch = text.charAt(i);
if (ch == ' ' || ch == '\n' || ch == '(') {
if (Character.isWhitespace(ch) || ch == '(') {
scriptHeadIndex = i;
break;
}

22
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -41,12 +41,34 @@ public class ParagraphTest {
text = "%table 1234567";
assertEquals("1234567", Paragraph.getScriptBody(text));
}
@Test
public void scriptBodyWithoutReplName() {
String text = "12345678";
assertEquals(text, Paragraph.getScriptBody(text));
}
@Test
public void replNameEndsWithWhitespace() {
String text = "%md\r\n###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\t###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\u000b###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\f###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\n###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md ###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given