Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-732-up

Browse source
This commit is contained in:
Lee moon soo 2016-05-28 13:17:49 -07:00
commit 99dd0b0b4e
74 changed files with 4457 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -25,6 +25,7 @@ conf/keystore
conf/truststore
conf/interpreter.json
conf/notebook-authorization.json
conf/shiro.ini
# other generated files
spark/dependency-reduced-pom.xml
@ -86,6 +87,9 @@ Thumbs.db
target/
**/target/
# Generated by Jekyll
docs/_site/
*~
\#*\#
/.emacs.desktop

6
conf/zeppelin-env.cmd.template
View file

@ -34,6 +34,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM Spark interpreter configuration
@ -62,3 +63,8 @@ REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext
REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default.
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default.
REM ZeppelinHub connection configuration
REM
REM set ZEPPELINHUB_API_ADDRESS REM Refers to the address of the ZeppelinHub service in use
REM set ZEPPELINHUB_API_TOKEN REM Refers to the Zeppelin instance token of the user
REM set ZEPPELINHUB_USER_KEY REM Optional, when using Zeppelin with authentication.

8
conf/zeppelin-env.sh.template
View file

@ -35,7 +35,7 @@
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
#### Spark interpreter configuration ####
@ -70,3 +70,9 @@
# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be
# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml
#### ZeppelinHub connection configuration ####
# export ZEPPELINHUB_API_ADDRESS # Refers to the address of the ZeppelinHub service in use
# export ZEPPELINHUB_API_TOKEN # Refers to the Zeppelin instance token of the user
# export ZEPPELINHUB_USER_KEY # Optional, when using Zeppelin with authentication.

52
conf/zeppelin-site.xml.template
View file

@ -62,7 +62,8 @@
</property>
<!-- If used S3 to storage the notebooks, it is necessary the following folder structure bucketname/username/notebook/ -->
<!-- Amazon S3 notebook storage -->
<!-- Creates the following directory structure: s3://{bucket}/{username}/{notebook-id}/note.json -->
<!--
<property>
<name>zeppelin.notebook.s3.user</name>
@ -89,12 +90,36 @@
</property>
-->
<!-- Additionally, encryption is supported for notebook data stored in S3 -->
<!-- Use the AWS KMS to encrypt data -->
<!-- If used, the EC2 role assigned to the EMR cluster must have rights to use the given key -->
<!-- See https://aws.amazon.com/kms/ and http://docs.aws.amazon.com/kms/latest/developerguide/concepts.html -->
<!--
<property>
<name>zeppelin.notebook.s3.kmsKeyID</name>
<value>AWS-KMS-Key-UUID</value>
<description>AWS KMS key ID used to encrypt notebook data in S3</description>
</property>
-->
<!-- Use a custom encryption materials provider to encrypt data -->
<!-- No configuration is given to the provider, so you must use system properties or another means to configure -->
<!-- See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html -->
<!--
<property>
<name>zeppelin.notebook.s3.encryptionMaterialsProvider</name>
<value>provider implementation class name</value>
<description>Custom encryption materials provider used to encrypt notebook data in S3</description>
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>
<name>zeppelin.notebook.azure.user</name>
<value>user</value>
<description>optional user name for Azure folder structure</description>
<name>zeppelin.notebook.azure.connectionString</name>
<value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
<description>Azure account credentials</description>
</property>
<property>
@ -104,9 +129,9 @@
</property>
<property>
<name>zeppelin.notebook.azure.connectionString</name>
<value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
<description>share name for notebook storage</description>
<name>zeppelin.notebook.azure.user</name>
<value>user</value>
<description>optional user name for Azure folder structure</description>
</property>
<property>
@ -116,7 +141,7 @@
</property>
-->
<!-- For versioning your local norebook storage using Git repository
<!-- For versioning your local notebook storage using Git repository
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
@ -124,6 +149,15 @@
</property>
-->
<!-- For connecting your Zeppelin with ZeppelinHub -->
<!--
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo</value>
<description>two notebook persistence layers (local + ZeppelinHub)</description>
</property>
-->
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
@ -144,7 +178,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

9
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -54,6 +54,7 @@
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
<li><a href="{{BASE_PATH}}/interpreter/r.html">R</a></li>
@ -85,6 +86,8 @@
<!-- li><span><b>Notebook Storage</b><span></li -->
<li><a href="{{BASE_PATH}}/storage/storage.html#Git">Git Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#S3">S3 Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#Azure">Azure Storage</a></li>
<li><a href="{{BASE_PATH}}/storage/storage.html#ZeppelinHub">ZeppelinHub Storage</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>REST API</b><span></li -->
<li><a href="{{BASE_PATH}}/rest-api/rest-interpreter.html">Interpreter API</a></li>
@ -93,7 +96,8 @@
<li role="separator" class="divider"></li>
<!-- li><span><b>Security</b><span></li -->
<li><a href="{{BASE_PATH}}/security/overview.html">Security Overview</a></li>
<li><a href="{{BASE_PATH}}/security/authentication.html">Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/authentication.html">Authentication for NGINX</a></li>
<li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/interpreter_authorization.html">Interpreter Authorization</a></li>
<li role="separator" class="divider"></li>
@ -101,9 +105,6 @@
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Shiro Security</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/shiroauthentication.html">Shiro Authentication</a></li>
</ul>
</li>
</ul>

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin-login.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 245 KiB

6
docs/assets/themes/zeppelin/js/docs.js
View file

@ -99,9 +99,7 @@ function viewSolution() {
// A script to fix internal hash links because we have an overlapping top bar.
// Based on https://github.com/twitter/bootstrap/issues/193#issuecomment-2281510
function maybeScrollToHash() {
console.log("HERE");
if (window.location.hash && $(window.location.hash).length) {
console.log("HERE2", $(window.location.hash), $(window.location.hash).offset().top);
var newTop = $(window.location.hash).offset().top - 57;
$(window).scrollTop(newTop);
}
@ -117,5 +115,5 @@ $(function() {
// Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers
// will try to do *their* initial scroll after running the onReady handler.
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });
});
$(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); });
});

12
docs/install/install.md
View file

@ -192,6 +192,18 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>s3.amazonaws.com</td>
<td>Endpoint for the bucket</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID</td>
<td>zeppelin.notebook.s3.kmsKeyID</td>
<td></td>
<td>AWS KMS Key ID to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_EMP</td>
<td>zeppelin.notebook.s3.encryptionMaterialsProvider</td>
<td></td>
<td>Class name of a custom S3 encryption materials provider implementation to use for encrypting data in S3 (optional)</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td>
<td>zeppelin.notebook.azure.connectionString</td>

107
docs/interpreter/livy.md Normal file
View file

@ -0,0 +1,107 @@
---
layout: page
title: "Livy Interpreter"
description: ""
group: manual
---
{% include JB/setup %}
## Livy Interpreter for Apache Zeppelin
Livy is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN.
* Interactive Scala, Python and R shells
* Batch submissions in Scala, Java, Python
* Multi users can share the same server (impersonation support)
* Can be used for submitting jobs from anywhere with REST
* Does not require any code change to your programs
### Requirements
Additional requirements for the Livy interpreter are:
* Spark 1.3 or above.
* Livy server.
### Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
<th>Default</th>
<th>Description</th>
</tr>
<tr>
<td>zeppelin.livy.master</td>
<td>local[*]</td>
<td>Spark master uri. ex) spark://masterhost:7077</td>
</tr>
<tr>
<td>zeppelin.livy.url</td>
<td>http://localhost:8998</td>
<td>URL where livy server is running</td>
</tr>
<tr>
<td>zeppelin.livy.spark.maxResult</td>
<td>1000</td>
<td>Max number of SparkSQL result to display.</td>
</tr>
</table>
## How to use
Basically, you can use
**spark**
```
%livy.spark
sc.version
```
**pyspark**
```
%livy.pyspark
print "1"
```
**sparkR**
```
%livy.sparkr
hello <- function( name ) {
sprintf( "Hello, %s", name );
}
hello("livy")
```
## Impersonation
When Zeppelin server is running with authentication enabled, then this interpreter utilizes Livy’s user impersonation feature i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server.
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features.
```
%livy.pyspark
print "${group_by=product_id,product_id|product_name|customer_id|store_id}"
```
## FAQ
Livy debugging: If you see any of these in error console
> Connect to livyhost:8998 [livyhost/127.0.0.1, livyhost/0:0:0:0:0:0:0:1] failed: Connection refused
Looks like the livy server is not up yet or the config is wrong
> Exception: Session not found, Livy server would have restarted, or lost session.
The session would have timed out, you may need to restart the interpreter.
> Blacklisted configuration values in session config: spark.master
edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line.

10
docs/manual/interpreters.md
View file

@ -29,18 +29,18 @@ Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_create.png">
## What is Zeppelin Interpreter Setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
Each notebook can be bound to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
@ -51,7 +51,7 @@ By default, every interpreter is belonged to a single group, but the group might
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
## Interpreter binding mode
@ -59,4 +59,4 @@ Each interpreters is belonged to a single group and registered together. All of
Each Interpreter Setting can choose one of 'shared', 'scoped', 'isolated' interpreter binding mode.
In 'shared' mode, every notebook bound to the Interpreter Setting will share the single Interpreter instance. In 'scoped' mode, each notebook will create new Interpreter instance in the same interpreter process. In 'isolated' mode, each notebook will create new Interpreter process.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">

6
docs/security/authentication.md
View file

@ -1,7 +1,7 @@
---
layout: page
title: "Authentication"
description: "Authentication"
title: "Authentication for NGINX"
description: "Authentication for NGINX"
group: security
---
<!--
@ -17,7 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Authentication
# Authentication for NGINX
Authentication is company-specific.

4
docs/manual/shiroauthentication.md → docs/security/shiroauthentication.md
View file

@ -2,7 +2,7 @@
layout: page
title: "Shiro Security for Apache Zeppelin"
description: ""
group: manual
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
@ -59,7 +59,7 @@ Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
```
admin = password1

174
docs/storage/storage.md
View file

@ -20,15 +20,17 @@ limitations under the License.
### Notebook Storage
Zeppelin has a pluggable notebook storage mechanism controlled by `zeppelin.notebook.storage` configuration option with multiple implementations.
There are few Notebook storages available for a use out of the box:
There are few Notebook storage systems available for a use out of the box:
- (default) all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
- there is also an option to version it using local Git repository - `GitNotebookRepo`
- another option is Amazon S3 service - `S3NotebookRepo`
- use local file system and version it using local Git repository - `GitNotebookRepo`
- storage using Amazon S3 service - `S3NotebookRepo`
- storage using Azure service - `AzureNotebookRepo`
Multiple storages can be used at the same time by providing a comma-separated list of the class-names in the configuration.
Multiple storage systems can be used at the same time by providing a comma-separated list of the class-names in the configuration.
By default, only first two of them will be automatically kept in sync by Zeppelin.
</br>
#### Notebook Storage in local Git repository <a name="Git"></a>
To enable versioning for all your local notebooks though a standard Git repository - uncomment the next property in `zeppelin-site.xml` in order to use GitNotebookRepo class:
@ -42,44 +44,46 @@ To enable versioning for all your local notebooks though a standard Git reposito
```
</br>
#### Notebook Storage in S3 <a name="S3"></a>
For notebook storage in S3 you need the AWS credentials, for this there are three options, the environment variable ```AWS_ACCESS_KEY_ID``` and ```AWS_ACCESS_SECRET_KEY```, credentials file in the folder .aws in you home and IAM role for your instance. For complete the need steps is necessary:
Notebooks may be stored in S3, and optionally encrypted. The [``DefaultAWSCredentialsProviderChain``](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) credentials provider is used for credentials and checks the following:
- The ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment variables
- The ``aws.accessKeyId`` and ``aws.secretKey`` Java System properties
- Credential profiles file at the default location (````~/.aws/credentials````) used by the AWS CLI
- Instance profile credentials delivered through the Amazon EC2 metadata service
</br>
you need the following folder structure on S3
The following folder structure will be created in S3:
```
bucket_name/
username/
notebook/
s3://bucket_name/username/notebook-id/
```
set the environment variable in the file **zeppelin-env.sh**:
Configure by setting environment variables in the file **zeppelin-env.sh**:
```
export ZEPPELIN_NOTEBOOK_S3_BUCKET = bucket_name
export ZEPPELIN_NOTEBOOK_S3_USER = username
```
in the file **zeppelin-site.xml** uncomment and complete the next property:
Or using the file **zeppelin-site.xml** uncomment and complete the S3 settings:
```
<!--If used S3 to storage, it is necessary the following folder structure bucket_name/username/notebook/-->
<property>
<name>zeppelin.notebook.s3.user</name>
<value>username</value>
<description>user name for s3 folder structure</description>
</property>
<property>
<name>zeppelin.notebook.s3.bucket</name>
<value>bucket_name</value>
<description>bucket name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.s3.user</name>
<value>username</value>
<description>user name for s3 folder structure</description>
</property>
```
uncomment the next property for use S3NotebookRepo class:
Uncomment the next property for use S3NotebookRepo class:
```
<property>
@ -89,7 +93,7 @@ uncomment the next property for use S3NotebookRepo class:
</property>
```
comment the next property:
Comment out the next property to disable local notebook storage (the default):
```
<property>
@ -97,4 +101,134 @@ comment the next property:
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
```
#### Data Encryption in S3
##### AWS KMS encryption keys
To use an [AWS KMS](https://aws.amazon.com/kms/) encryption key to encrypt notebooks, set the following environment variable in the file **zeppelin-env.sh**:
```
export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID = kms-key-id
```
Or using the following setting in **zeppelin-site.xml**:
```
<property>
<name>zeppelin.notebook.s3.kmsKeyID</name>
<value>AWS-KMS-Key-UUID</value>
<description>AWS KMS key ID used to encrypt notebook data in S3</description>
</property>
```
##### Custom Encryption Materials Provider class
You may use a custom [``EncryptionMaterialsProvider``](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html) class as long as it is available in the classpath and able to initialize itself from system properties or another mechanism. To use this, set the following environment variable in the file **zeppelin-env.sh**:
```
export ZEPPELIN_NOTEBOOK_S3_EMP = class-name
```
Or using the following setting in **zeppelin-site.xml**:
```
<property>
<name>zeppelin.notebook.s3.encryptionMaterialsProvider</name>
<value>provider implementation class name</value>
<description>Custom encryption materials provider used to encrypt notebook data in S3</description>
```
</br>
#### Notebook Storage in Azure <a name="Azure"></a>
Using `AzureNotebookRepo` you can connect your Zeppelin with your Azure account for notebook storage.
</br>
First of all, input your `AccountName`, `AccountKey`, and `Share Name` in the file **zeppelin-site.xml** by commenting out and completing the next properties:
```
<property>
<name>zeppelin.notebook.azure.connectionString</name>
<value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
<description>Azure account credentials</description>
</property>
<property>
<name>zeppelin.notebook.azure.share</name>
<value>zeppelin</value>
<description>share name for notebook storage</description>
</property>
```
Secondly, you can initialize `AzureNotebookRepo` class in the file **zeppelin-site.xml** by commenting the next property:
```
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
```
and commenting out:
```
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.AzureNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
```
In case you want to use simultaneously your local storage with Azure storage use the following property instead:
```
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo, apache.zeppelin.notebook.repo.AzureNotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
```
Optionally, you can specify Azure folder structure name in the file **zeppelin-site.xml** by commenting out the next property:
```
<property>
<name>zeppelin.notebook.azure.user</name>
<value>user</value>
<description>optional user name for Azure folder structure</description>
</property>
```
</br>
#### Notebook Storage in ZeppelinHub <a name="ZeppelinHub"></a>
ZeppelinHub storage layer allows out of the box connection of Zeppelin instance with your ZeppelinHub account. First of all, you need to either comment out the following property in **zeppelin-site.xml**:
```
<!-- For connecting your Zeppelin with ZeppelinHub -->
<!--
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo</value>
<description>two notebook persistence layers (local + ZeppelinHub)</description>
</property>
-->
```
or set the environment variable in the file **zeppelin-env.sh**:
```
export ZEPPELIN_NOTEBOOK_STORAGE="org.apache.zeppelin.notebook.repo.VFSNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo"
```
Secondly, you need to set the environment variables in the file **zeppelin-env.sh**:
```
export ZEPPELINHUB_API_TOKEN = ZeppelinHub token
export ZEPPELINHUB_API_ADDRESS = address of ZeppelinHub service (e.g. https://www.zeppelinhub.com)
```
You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).

55
docs/ui_layout/zeppelin_layout.md
View file

@ -21,21 +21,21 @@ limitations under the License.
The first time you connect to Zeppelin, you'll land at the main page similar to the below screen capture
<img src="/assets/themes/zeppelin/img/ui-img/homepage.png" />
<img src="../assets/themes/zeppelin/img/ui-img/homepage.png" />
On the left of the page are listed all existing notes. Those notes are stored by default in the `$ZEPPELIN_HOME/notebook` folder.
On the left of the page are listed all existing notes. Those notes are stored by default in the `$ZEPPELIN_HOME/notebook` folder.
You can filter them by name using the input text form. You can also create an new note, refresh the list of existing notes
You can filter them by name using the input text form. You can also create an new note, refresh the list of existing notes
(in case you manually copy them into the `$ZEPPELIN_HOME/notebook` folder) and import a note
<img src="/assets/themes/zeppelin/img/ui-img/notes_management.png" />
<img src="../assets/themes/zeppelin/img/ui-img/notes_management.png" />
When clicking on `Import Note` link, a new dialog open. From there you can import your note from local disk or from a remote location
if you provide the URL.
<img src="/assets/themes/zeppelin/img/ui-img/note_import_dialog.png" />
By default, the name of the imported note is the same as the original note but you can override it by providing a new name
<img src="../assets/themes/zeppelin/img/ui-img/note_import_dialog.png" />
By default, the name of the imported note is the same as the original note but you can override it by providing a new name
<br />
## Menus
@ -67,10 +67,10 @@ This menu displays all the Zeppelin configuration that are set in the config fil
<br />
## Note Layout
## Note Layout
Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a paragraph container.
Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a paragraph container.
<img src="../assets/themes/zeppelin/img/ui-img/note_paragraph_layout.png" />
### Paragraph
@ -78,20 +78,20 @@ Each Zeppelin note is composed of 1 .. N paragraphs. The note can be viewed as a
Each paragraph consists of 2 sections: `code section` where you put your source code and `result section` where you can see the result of the code execution.
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_layout.png" />
On the top-right corner of each paragraph there are some commands to:
* execute the paragraph code
* hide/show `code section`
* hide/show `result section`
* configure the paragraph
To configure the paragraph, just click on the gear icon:
<img src="../assets/themes/zeppelin/img/ui-img/paragraph_configuration_dialog.png" />
From this dialog, you can (in descending order):
* find the **paragraph id** ( **20150924-163507_134879501** )
* control paragraph width. Since Zeppelin is using the grid system of **Twitter Bootstrap**, each paragraph width can be changed from 1 to 12
* move the paragraph 1 level up
@ -103,19 +103,19 @@ From this dialog, you can (in descending order):
* export the current paragraph as an **iframe** and open the **iframe** in a new window
* clear the `result section`
* delete the current paragraph
### Note toolbar
At the top of the note, you can find a toolbar which exposes command buttons as well as configuration, security and display options
<img src="../assets/themes/zeppelin/img/ui-img/note_toolbar.png" />
<img src="../assets/themes/zeppelin/img/ui-img/note_toolbar.png" />
On the far right is displayed the note name, just click on it to reveal the input form and update it
In the middle of the toolbar you can find the command buttons:
* execute all the paragraphs **sequentially**, in their display order
* hide/show `code section` of all paragraphs
* hide/show `code section` of all paragraphs
* hide/show `result section` of all paragraphs
* clear the `result section` of all paragraphs
* clone the current note
@ -127,13 +127,10 @@ In the middle of the toolbar you can find the command buttons:
<img src="../assets/themes/zeppelin/img/ui-img/note_commands.png" />
On the right of the note tool bar you can find configuration icons:
* display all the keyboard shorcuts
* configure the interpreters binding to the current note
* configure the note permissions
* switch the node display mode between `default`, `simple` and `report`
<img src="../assets/themes/zeppelin/img/ui-img/note_configuration.png" />

8
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -191,13 +191,7 @@ public class JDBCInterpreter extends Interpreter {
logger.info(properties.getProperty(DRIVER_KEY));
Class.forName(properties.getProperty(DRIVER_KEY));
String url = properties.getProperty(URL_KEY);
String user = properties.getProperty(USER_KEY);
String password = properties.getProperty(PASSWORD_KEY);
if (null != user && null != password) {
connection = DriverManager.getConnection(url, user, password);
} else {
connection = DriverManager.getConnection(url, properties);
}
connection = DriverManager.getConnection(url, properties);
}
return connection;
}

170
livy/pom.xml Normal file
View file

@ -0,0 +1,170 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-livy</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Livy interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<!--TEST-->
<junit.version>4.12</junit.version>
<achilles.version>3.2.4-Zeppelin</achilles.version>
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.4</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/livy
</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

409
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java Normal file
View file

@ -0,0 +1,409 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/***
* Livy helper class
*/
public class LivyHelper {
Logger LOGGER = LoggerFactory.getLogger(LivyHelper.class);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
HashMap<String, Object> paragraphHttpMap = new HashMap<>();
Properties property;
Integer MAX_NOS_RETRY = 60;
LivyHelper(Properties property) {
this.property = property;
}
public Integer createSession(InterpreterContext context, String kind) throws Exception {
try {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"master\": \"" + property.getProperty("zeppelin.livy.master") + "\", " +
"\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
"}",
context.getParagraphId()
);
if (json.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"conf\":{\"spark.master\": \""
+ property.getProperty("zeppelin.livy.master") + "\"}," +
"\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" +
"}",
context.getParagraphId()
);
}
Map jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
Integer sessionId = ((Double) jsonMap.get("id")).intValue();
if (!jsonMap.get("state").equals("idle")) {
Integer nosRetry = MAX_NOS_RETRY;
while (nosRetry >= 0) {
LOGGER.error(String.format("sessionId:%s state is %s",
jsonMap.get("id"), jsonMap.get("state")));
Thread.sleep(1000);
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId,
"GET", null,
context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
if (jsonMap.get("state").equals("idle")) {
break;
} else if (jsonMap.get("state").equals("error")) {
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId + "/log",
"GET", null,
context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());
String logs = StringUtils.join((ArrayList<String>) jsonMap.get("log"), '\n');
LOGGER.error(String.format("Cannot start %s.\n%s", kind, logs));
throw new Exception(String.format("Cannot start %s.\n%s", kind, logs));
}
nosRetry--;
}
if (nosRetry <= 0) {
LOGGER.error("Error getting session for user within 60Sec.");
throw new Exception(String.format("Cannot start %s.", kind));
}
}
return sessionId;
} catch (Exception e) {
LOGGER.error("Error getting session for user", e);
throw e;
}
}
protected void initializeSpark(final InterpreterContext context,
final Map<String, Integer> userSessionMap) throws Exception {
interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" +
"import sqlContext.implicits._", context, userSessionMap);
}
public InterpreterResult interpretInput(String stringLines,
final InterpreterContext context,
final Map<String, Integer> userSessionMap,
LivyOutputStream out) {
try {
String[] lines = stringLines.split("\n");
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
out.setInterpreterOutput(context.out);
context.out.clear();
Code r = null;
String incomplete = "";
boolean inComment = false;
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
//for spark
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
boolean continuation = false;
if (nextLine.isEmpty()
|| nextLine.startsWith("//") // skip empty line or comment
|| nextLine.startsWith("}")
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
continuation = true;
} else if (!inComment && nextLine.startsWith("/*")) {
inComment = true;
continuation = true;
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
inComment = false;
continuation = true;
} else if (nextLine.length() > 1
&& nextLine.charAt(0) == '.'
&& nextLine.charAt(1) != '.' // ".."
&& nextLine.charAt(1) != '/') { // "./"
continuation = true;
} else if (inComment) {
continuation = true;
}
if (continuation) {
incomplete += s + "\n";
continue;
}
}
InterpreterResult res;
try {
res = interpret(incomplete + s, context, userSessionMap);
} catch (Exception e) {
LOGGER.error("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = res.code();
if (r == Code.ERROR) {
out.setInterpreterOutput(null);
return res;
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
out.write((res.message() + "\n").getBytes(Charset.forName("UTF-8")));
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
out.setInterpreterOutput(null);
return new InterpreterResult(r, "Incomplete expression");
} else {
out.setInterpreterOutput(null);
return new InterpreterResult(Code.SUCCESS);
}
} catch (Exception e) {
LOGGER.error("error in interpretInput", e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
public InterpreterResult interpret(String stringLines,
final InterpreterContext context,
final Map<String, Integer> userSessionMap)
throws Exception {
stringLines = stringLines
//for "\n" present in string
.replaceAll("\\\\n", "\\\\\\\\n")
//for new line present in string
.replaceAll("\\n", "\\\\n")
// for \" present in string
.replaceAll("\\\\\"", "\\\\\\\\\"")
// for " present in string
.replaceAll("\"", "\\\\\"");
if (stringLines.trim().equals("")) {
return new InterpreterResult(Code.SUCCESS, "");
}
Map jsonMap = executeCommand(stringLines, context, userSessionMap);
Integer id = ((Double) jsonMap.get("id")).intValue();
InterpreterResult res = getResultFromMap(jsonMap);
if (res != null) {
return res;
}
while (true) {
Thread.sleep(1000);
if (paragraphHttpMap.get(context.getParagraphId()) == null) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
jsonMap = getStatusById(context, userSessionMap, id);
InterpreterResult interpreterResult = getResultFromMap(jsonMap);
if (interpreterResult != null) {
return interpreterResult;
}
}
}
private InterpreterResult getResultFromMap(Map jsonMap) {
if (jsonMap.get("state").equals("available")) {
if (((Map) jsonMap.get("output")).get("status").equals("error")) {
StringBuilder errorMessage = new StringBuilder((String) ((Map) jsonMap
.get("output")).get("evalue"));
if (errorMessage.toString().equals("incomplete statement")
|| errorMessage.toString().contains("EOF")) {
return new InterpreterResult(Code.INCOMPLETE, "");
}
String traceback = gson.toJson(((Map) jsonMap.get("output")).get("traceback"));
if (!traceback.equals("[]")) {
errorMessage
.append("\n")
.append("traceback: \n")
.append(traceback);
}
return new InterpreterResult(Code.ERROR, errorMessage.toString());
}
if (((Map) jsonMap.get("output")).get("status").equals("ok")) {
String result = (String) ((Map) ((Map) jsonMap.get("output"))
.get("data")).get("text/plain");
if (result != null) {
result = result.trim();
if (result.startsWith("<link")
|| result.startsWith("<script")
|| result.startsWith("<style")
|| result.startsWith("<div")) {
result = "%html " + result;
}
}
return new InterpreterResult(Code.SUCCESS, result);
}
}
return null;
}
private Map executeCommand(String lines, InterpreterContext context,
Map<String, Integer> userSessionMap) throws Exception {
String json = executeHTTP(property.get("zeppelin.livy.url") + "/sessions/"
+ userSessionMap.get(context.getAuthenticationInfo().getUser())
+ "/statements",
"POST",
"{\"code\": \"" + lines + "\" }",
context.getParagraphId());
if (json.matches("^(\")?Session (\'[0-9]\' )?not found(.?\"?)$")) {
throw new Exception("Exception: Session not found, Livy server would have restarted, " +
"or lost session.");
}
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error executeCommand", e);
throw e;
}
}
private Map getStatusById(InterpreterContext context,
Map<String, Integer> userSessionMap, Integer id) throws Exception {
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
+ userSessionMap.get(context.getAuthenticationInfo().getUser())
+ "/statements/" + id,
"GET", null, context.getParagraphId());
try {
Map jsonMap = gson.fromJson(json,
new TypeToken<Map>() {
}.getType());
return jsonMap;
} catch (Exception e) {
LOGGER.error("Error getStatusById", e);
throw e;
}
}
protected String executeHTTP(String targetURL, String method, String jsonData, String paragraphId)
throws Exception {
HttpClient client = HttpClientBuilder.create().build();
HttpResponse response = null;
if (method.equals("POST")) {
HttpPost request = new HttpPost(targetURL);
request.addHeader("Content-Type", "application/json");
StringEntity se = new StringEntity(jsonData);
request.setEntity(se);
response = client.execute(request);
paragraphHttpMap.put(paragraphId, request);
} else if (method.equals("GET")) {
HttpGet request = new HttpGet(targetURL);
request.addHeader("Content-Type", "application/json");
response = client.execute(request);
paragraphHttpMap.put(paragraphId, request);
} else if (method.equals("DELETE")) {
HttpDelete request = new HttpDelete(targetURL);
request.addHeader("Content-Type", "application/json");
response = client.execute(request);
}
if (response == null) {
return null;
}
if (response.getStatusLine().getStatusCode() == 200
|| response.getStatusLine().getStatusCode() == 201
|| response.getStatusLine().getStatusCode() == 404) {
return getResponse(response);
} else {
String responseString = getResponse(response);
if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
return responseString;
}
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusLine().getStatusCode(), responseString));
throw new Exception(String.format("Error with %s StatusCode: %s",
response.getStatusLine().getStatusCode(), responseString));
}
}
private String getResponse(HttpResponse response) throws Exception {
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
return result.toString();
}
public void cancelHTTP(String paragraphId) {
if (paragraphHttpMap.get(paragraphId).getClass().getName().contains("HttpPost")) {
((HttpPost) paragraphHttpMap.get(paragraphId)).abort();
} else {
((HttpGet) paragraphHttpMap.get(paragraphId)).abort();
}
paragraphHttpMap.put(paragraphId, null);
}
public void closeSession(Map<String, Integer> userSessionMap) {
for (Map.Entry<String, Integer> entry : userSessionMap.entrySet()) {
try {
executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/"
+ entry.getValue(),
"DELETE", null, null);
} catch (Exception e) {
LOGGER.error(String.format("Error closing session for user with session ID: %s",
entry.getValue()), e);
}
}
}
}

75
livy/src/main/java/org/apache/zeppelin/livy/LivyOutputStream.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import java.io.IOException;
import java.io.OutputStream;
/**
* InterpreterOutput can be attached / detached.
*/
public class LivyOutputStream extends OutputStream {
InterpreterOutput interpreterOutput;
public LivyOutputStream() {
}
public InterpreterOutput getInterpreterOutput() {
return interpreterOutput;
}
public void setInterpreterOutput(InterpreterOutput interpreterOutput) {
this.interpreterOutput = interpreterOutput;
}
@Override
public void write(int b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte[] b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte[] b, int offset, int len) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b, offset, len);
}
}
@Override
public void close() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.close();
}
}
@Override
public void flush() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.flush();
}
}
}

122
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivyPySparkInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivyPySparkInterpreter.class);
static {
Interpreter.register(
"pyspark",
"livy",
LivyPySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
protected LivyHelper livyHelper;
public LivyPySparkInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"pyspark")
);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivyPySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

135
livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java Normal file
View file

@ -0,0 +1,135 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy Spark interpreter for Zeppelin.
*/
public class LivySparkInterpreter extends Interpreter {
static String DEFAULT_URL = "http://localhost:8998";
static String LOCAL = "local[*]";
Logger LOGGER = LoggerFactory.getLogger(LivySparkInterpreter.class);
private LivyOutputStream out;
static {
Interpreter.register(
"spark",
"livy",
LivySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.")
.add("zeppelin.livy.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
.build()
);
}
protected static Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
out = new LivyOutputStream();
}
protected static Map<String, Integer> getUserSessionMap() {
return userSessionMap;
}
public void setUserSessionMap(Map<String, Integer> userSessionMap) {
this.userSessionMap = userSessionMap;
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"spark")
);
livyHelper.initializeSpark(interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpretInput(line, interpreterContext, userSessionMap, out);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

122
livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkRInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkRInterpreter.class);
static {
Interpreter.register(
"sparkr",
"livy",
LivySparkRInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkRInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"sparkr")
);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkRInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

165
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java Normal file
View file

@ -0,0 +1,165 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
static String DEFAULT_MAX_RESULT = "1000";
static {
Interpreter.register(
"sql",
"livy",
LivySparkSQLInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.spark.maxResult",
DEFAULT_MAX_RESULT,
"Max number of SparkSQL result to display.")
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkSQLInterpreter(Properties property) {
super(property);
livyHelper = new LivyHelper(property);
userSessionMap = LivySparkInterpreter.getUserSessionMap();
}
@Override
public void open() {
}
@Override
public void close() {
livyHelper.closeSession(userSessionMap);
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext,
"spark")
);
livyHelper.initializeSpark(interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
InterpreterResult res = livyHelper.interpret("sqlContext.sql(\"" +
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
property.get("zeppelin.livy.spark.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {
StringBuilder resMsg = new StringBuilder();
resMsg.append("%table ");
String[] rows = res.message().split("\n");
String[] headers = rows[1].split("\\|");
for (int head = 1; head < headers.length; head++) {
resMsg.append(headers[head].trim()).append("\t");
}
resMsg.append("\n");
if (rows[3].indexOf("+") == 0) {
} else {
for (int cols = 3; cols < rows.length - 1; cols++) {
String[] col = rows[cols].split("\\|");
for (int data = 1; data < col.length; data++) {
resMsg.append(col[data].trim()).append("\t");
}
resMsg.append("\n");
}
}
if (rows[rows.length - 1].indexOf("only") == 0) {
resMsg.append("<font color=red>" + rows[rows.length - 1] + ".</font>");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
resMsg.toString()
);
} else {
return res;
}
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

114
livy/src/main/test/org/apache/zeppelin/livy/LivyHelperTest.java Normal file
View file

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import com.google.gson.GsonBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.hamcrest.CoreMatchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Properties;
import static org.mockito.Mockito.doReturn;
/**
* Created for org.apache.zeppelin.livy on 22/04/16.
*/
@RunWith(MockitoJUnitRunner.class)
public class LivyHelperTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private static LivyPySparkInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext interpreterContext;
@Mock(answer = Answers.CALLS_REAL_METHODS)
private LivyHelper livyHelper;
@Before
public void prepareContext() throws Exception {
interpreter.userSessionMap = new HashMap<>();
interpreter.userSessionMap.put(null, 1);
Properties properties = new Properties();
properties.setProperty("zeppelin.livy.url", "http://localhost:8998");
livyHelper.property = properties;
livyHelper.paragraphHttpMap = new HashMap<>();
livyHelper.gson = new GsonBuilder().setPrettyPrinting().create();
doReturn("{\"id\":1,\"state\":\"idle\",\"kind\":\"spark\",\"proxyUser\":\"null\",\"log\":[]}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"{\"kind\": \"spark\", \"proxyUser\": \"null\"}",
null
);
doReturn("{\"id\":1,\"state\":\"available\",\"output\":{\"status\":\"ok\"," +
"\"execution_count\":1,\"data\":{\"text/plain\":\"1\"}}}")
.when(livyHelper)
.executeHTTP(
livyHelper.property.getProperty("zeppelin.livy.url") + "/sessions/1/statements",
"POST",
"{\"code\": \"print(1)\" }",
null
);
}
@Test
public void checkCreateSession() {
try {
Integer sessionId = livyHelper.createSession(interpreterContext, "spark");
collector.checkThat("check sessionId", 1, CoreMatchers.equalTo(sessionId));
} catch (Exception e) {
collector.addError(e);
}
}
@Test
public void checkInterpret() {
try {
InterpreterResult result = livyHelper.interpret("print(1)", interpreterContext, interpreter.userSessionMap);
collector.checkThat("check sessionId", InterpreterResult.Code.SUCCESS, CoreMatchers.equalTo(result.code()));
} catch (Exception e) {
collector.addError(e);
}
}
}

86
livy/src/main/test/org/apache/zeppelin/livy/LivyInterpreterTest.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Properties;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doReturn;
@RunWith(MockitoJUnitRunner.class)
public class LivyInterpreterTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
private static LivyPySparkInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext interpreterContext;
@AfterClass
public static void tearDown() {
interpreter.close();
}
@Before
public void prepareContext() throws Exception {
interpreter = new LivyPySparkInterpreter(new Properties());
interpreter.userSessionMap = new HashMap<>();
interpreter.userSessionMap.put(null, 0);
interpreter.livyHelper = Mockito.mock(LivyHelper.class);
interpreter.open();
doReturn(new InterpreterResult(InterpreterResult.Code.SUCCESS)).when(interpreter.livyHelper)
.interpret("print \"x is 1.\"", interpreterContext, interpreter.userSessionMap);
}
@Test
public void checkInitVariables() throws Exception {
collector.checkThat("Check that, if userSessionMap is made: ",
interpreter.userSessionMap, CoreMatchers.notNullValue());
}
@Test
public void checkBasicInterpreter() throws Exception {
String paragraphString = "print \"x is 1.\"";
final InterpreterResult actual = interpreter.interpret(paragraphString, interpreterContext);
collector.checkThat("Check that, result is computed: ",
actual.code(), CoreMatchers.equalTo(InterpreterResult.Code.SUCCESS));
assertThat(actual).isNotNull();
}
}

2
pom.xml
View file

@ -92,6 +92,7 @@
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>livy</module>
<module>hive</module>
<module>hbase</module>
<module>phoenix</module>
@ -117,6 +118,7 @@
<libthrift.version>0.9.2</libthrift.version>
<gson.version>2.2</gson.version>
<guava.version>15.0</guava.version>
<jetty.version>9.2.15.v20160210</jetty.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>

12
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -278,15 +278,21 @@ public class SparkInterpreter extends Interpreter {
classServerUri = (String) classServer.invoke(interpreter.intp());
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
throw new InterpreterException(e);
// continue instead of: throw new InterpreterException(e);
// Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method
logger.warn(String.format("Spark method classServerUri not available due to: [%s]",
e.getMessage()));
}
}
SparkConf conf =
new SparkConf()
.setMaster(getProperty("master"))
.setAppName(getProperty("spark.app.name"))
.set("spark.repl.class.uri", classServerUri);
.setAppName(getProperty("spark.app.name"));
if (classServerUri != null) {
conf.set("spark.repl.class.uri", classServerUri);
}
if (jars.length > 0) {
conf.setJars(jars);

7
zeppelin-server/pom.xml
View file

@ -35,7 +35,6 @@
<properties>
<cxf.version>2.7.7</cxf.version>
<jetty.version>9.2.15.v20160210</jetty.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
</properties>
@ -184,9 +183,9 @@
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
<scope>test</scope>
<exclusions>
<exclusion>

6
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -249,9 +249,6 @@ public class ZeppelinServer extends Application {
webapp.setSessionHandler(new SessionHandler());
webapp.addServlet(cxfServletHolder, "/api/*");
webapp.addFilter(new FilterHolder(CorsFilter.class), "/*",
EnumSet.allOf(DispatcherType.class));
webapp.setInitParameter("shiroConfigLocations",
new File(conf.getShiroPath()).toURI().toString());
@ -285,6 +282,9 @@ public class ZeppelinServer extends Application {
webApp.addServlet(new ServletHolder(new DefaultServlet()), "/*");
contexts.addHandler(webApp);
webApp.addFilter(new FilterHolder(CorsFilter.class), "/*",
EnumSet.allOf(DispatcherType.class));
return webApp;
}

9
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -43,10 +44,11 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.socket.Message.OP;
import org.apache.zeppelin.ticket.TicketContainer;
import org.apache.zeppelin.utils.SecurityUtils;
import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
@ -62,7 +64,8 @@ public class NotebookServer extends WebSocketServlet implements
NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener,
RemoteInterpreterProcessListener, ApplicationEventListener {
private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class);
Gson gson = new Gson();
Gson gson = new GsonBuilder()
.setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
final Map<String, List<NotebookSocket>> noteSocketMap = new HashMap<>();
final Queue<NotebookSocket> connectedSockets = new ConcurrentLinkedQueue<>();
@ -361,7 +364,7 @@ public class NotebookServer extends WebSocketServlet implements
try {
notebook.reloadAllNotes();
} catch (IOException e) {
LOG.error("Fail to reload notes from repository");
LOG.error("Fail to reload notes from repository", e);
}
}

33
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
View file

@ -17,11 +17,13 @@
package org.apache.zeppelin.integration;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.hamcrest.CoreMatchers;
import org.junit.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
@ -194,19 +196,19 @@ public class ZeppelinIT extends AbstractZeppelinIT {
}
try {
// navigate to interpreter page
WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
WebElement interpreterLink = driver.findElement(By.xpath("//a[contains(.,'Interpreter')]"));
interpreterLink.click();
// add new dependency to spark interpreter
WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
driver.findElement(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
WebElement depArtifact = pollingWait(By.xpath("//input[@ng-model='setting.depArtifact']"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
String artifact = "org.apache.commons:commons-csv:1.1";
depArtifact.sendKeys(artifact);
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.findElement(By.xpath("//div[contains(@class,'box')][contains(.,'%spark')]//form//button[1]")).click();
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
driver.navigate().back();
createNewNote();
@ -230,20 +232,17 @@ public class ZeppelinIT extends AbstractZeppelinIT {
//delete created notebook for cleanup.
deleteTestNotebook(driver);
sleep(1000, true);
sleep(1000, false);
// reset dependency
interpreterLink.click();
sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"));
sleep(5000, true);
driver.findElement(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
WebElement testDepRemoveBtn = pollingWait(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"), MAX_IMPLICIT_WAIT);
testDepRemoveBtn.click();
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.findElement(By.xpath("//div[contains(@class,'box')][contains(.,'%spark')]//form//button[1]")).click();
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
} catch (Exception e) {
handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
}

3
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -30,9 +30,10 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.socket.Message.OP;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

3
zeppelin-web/.jshintrc
View file

@ -32,6 +32,7 @@
"ace": false,
"d3": false,
"BootstrapDialog": false,
"Handsontable": false
"Handsontable": false,
"moment": false
}
}

6
zeppelin-web/src/app/home/home.controller.js
View file

@ -37,7 +37,7 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
console.log('Error %o %o', status, data.message);
});
};
var initHome = function() {
websocketMsgSrv.getHomeNotebook();
getZeppelinVersion();
@ -76,8 +76,4 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
node.hidden = !node.hidden;
};
$rootScope.noteName = function(note) {
return arrayOrderingSrv.getNoteName(note);
};
});

4
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -29,7 +29,7 @@ limitations under the License.
style="width:180px">
<select class="form-control input-sm" ng-model="newInterpreterSetting.group"
ng-change="newInterpreterGroupChange()">
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'group'" value="{{availableInterpreter.group}}">
<option ng-repeat="availableInterpreter in availableInterpreters | unique: 'group'| orderBy: 'group'" value="{{availableInterpreter.group}}">
{{availableInterpreter.group}}
</option>
</select>
@ -69,7 +69,7 @@ limitations under the License.
<span>Interpreter for note</span>
</div>
<br />
<b>Properties</b>
<table class="table table-striped properties">
<tr>

13
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -37,6 +37,15 @@ limitations under the License.
Manage interpreters settings. You can create create / remove settings. Note can bind/unbind these interpreter settings.
</div>
</div>
<div class="input-group col-md-4" style="margin-top: 10px">
<input type="text" ng-model="searchInterpreter" class="form-control ng-pristine ng-untouched ng-valid ng-empty" placeholder="Search interpreters">
<span class="input-group-btn">
<button type="submit" class="btn btn-default" ng-disabled="!navbar.connected">
<i class="glyphicon glyphicon-search"></i>
</button>
</span>
</div>
</div>
<div class="row" ng-if="showRepositoryInfo">
@ -78,7 +87,7 @@ limitations under the License.
</div>
<div class="box width-full home"
ng-repeat="setting in interpreterSettings">
ng-repeat="setting in interpreterSettings | orderBy: 'group' | filter: searchInterpreter">
<div>
<div class="row interpreter">
<div class="col-md-12">
@ -144,7 +153,7 @@ limitations under the License.
<span>Interpreter for note</span>
</div>
<div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
<em>Currently there are no properties and dependencies set for this interpreter</em>
</div>

7
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -131,6 +131,13 @@ limitations under the License.
{{note.info.cron}}
</p>
</div>
<div>
<span>- Cron executing user (click enter in field to submit)</span>
<input type="text"
ng-model="note.config.cronExecutingUser"
ng-enter="setCronExecutingUser(note.config.cronExecutingUser)"
dropdown-input />
</div>
<div>
<span>- auto-restart interpreter on cron execution </span>
<input type="checkbox"

6
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -295,6 +295,12 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
$scope.setConfig();
};
/** Set the username of the user to be used to execute all notes in notebook **/
$scope.setCronExecutingUser = function(cronExecutingUser) {
$scope.note.config.cronExecutingUser = cronExecutingUser;
$scope.setConfig();
};
/** Set release resource for this note **/
$scope.setReleaseResource = function(value) {
$scope.note.config.releaseresource = value;

12
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -1026,18 +1026,22 @@ angular.module('zeppelinWebApp')
return '';
}
var user = 'anonymous';
var authInfo = pdata.authenticationInfo;
if (authInfo && authInfo.user) {
if (pdata.authenticationInfo !== null && !isEmpty(pdata.authenticationInfo.user)) {
user = pdata.authenticationInfo.user;
}
var dateUpdated = (pdata.dateUpdated === null) ? 'unknown' : pdata.dateUpdated;
var desc = 'Took ' + (timeMs/1000) + ' seconds. Last updated by ' + user + ' at time ' + dateUpdated + '.';
var desc = 'Took ' +
moment.duration(moment(pdata.dateFinished).diff(moment(pdata.dateStarted))).humanize() +
'. Last updated by ' + user + ' at ' + moment(pdata.dateUpdated).format('MMMM DD YYYY, h:mm:ss A') + '.';
if ($scope.isResultOutdated()){
desc += ' (outdated)';
}
return desc;
};
$scope.getElapsedTime = function() {
return 'Started ' + moment($scope.paragraph.dateStarted).fromNow() + '.';
};
$scope.isResultOutdated = function() {
var pdata = $scope.paragraph;
if (pdata.dateUpdated !==undefined && Date.parse(pdata.dateUpdated) > Date.parse(pdata.dateStarted)){

8
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -82,11 +82,13 @@ table.dataTable.table-condensed .sorting_desc:after {
border: 3px solid #DDDDDD;
}
.paragraph .paragraphFooter {
height: 9px;
.paragraph .executionTime {
color: #999;
font-size: 10px;
font-family: 'Roboto', sans-serif;
}
.paragraph .executionTime {
.paragraph .elapsedTime {
color: #999;
font-size: 10px;
font-family: 'Roboto', sans-serif;

6
zeppelin-web/src/app/notebook/paragraph/paragraph.html
View file

@ -68,5 +68,11 @@ limitations under the License.
id="{{paragraph.id}}_executionTime"
class="executionTime" ng-bind-html="getExecutionTime()">
</div>
<div ng-if = "paragraph.status === 'RUNNING'" class = "paragraphFooterElapsed">
<div
id="{{paragraph.id}}_elapsedTime"
class="elapsedTime" ng-bind-html="getElapsedTime()">
</div>
</div>
</div>
</div>

15
zeppelin-web/src/assets/styles/looknfeel/simple.css
View file

@ -55,9 +55,14 @@ body {
.paragraph .paragraphFooter {
visibility: hidden;
height: 0;
position: relative;
top : -13px;
top : -9px;
z-index: 99;
}
.paragraph .paragraphFooterElapsed {
height: 0px;
float: right;
z-index: 99;
}
@ -67,6 +72,12 @@ body {
margin-right: 5px;
}
.paragraph .elapsedTime {
font-size: 8px;
text-align: right;
margin-right: 5px;
}
.paragraph:hover .paragraphFooter {
visibility: visible;
}

19
zeppelin-web/src/components/navbar/navbar.controller.js
View file

@ -46,15 +46,6 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
vm.connected = param;
});
$rootScope.$on('$locationChangeSuccess', function () {
var path = $location.path();
// hacky solution to clear search bar
// TODO(felizbear): figure out how to make ng-click work in navbar
if (path === '/') {
$scope.searchTerm = '';
}
});
$scope.checkUsername = function () {
if ($rootScope.ticket) {
if ($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
@ -91,8 +82,8 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
});
};
$scope.search = function() {
$location.url(/search/ + $scope.searchTerm);
$scope.search = function(searchTerm) {
$location.url(/search/ + searchTerm);
};
function loadNotes() {
@ -103,6 +94,12 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
return ($routeParams.noteId === noteId);
}
$rootScope.noteName = function(note) {
if (!_.isEmpty(note)) {
return arrayOrderingSrv.getNoteName(note);
}
};
vm.loadNotes = loadNotes;
vm.isActive = isActive;

10
zeppelin-web/src/components/navbar/navbar.html
View file

@ -8,7 +8,8 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
-->
<script type="text/ng-template" id="notebook_list_renderer.html">
<a ng-if="note.id" href="#/notebook/{{note.id}}">{{noteName(note)}} </a>
<li ng-if="!note.id"
@ -43,10 +44,7 @@ limitations under the License.
<li class="divider"></li>
<div id="notebook-list" class="scrollbar-container">
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
<li ng-repeat="note in navbar.notes.list | filter:query | orderBy:navbar.arrayOrderingSrv.notebookListOrdering track by $index"
ng-class="{'active' : navbar.isActive(note.id)}">
<a href="#/notebook/{{note.id}}">{{noteName(note)}} </a>
<li ng-repeat="note in navbar.notes.root.children track by $index" ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'notebook_list_renderer.html'">
<li ng-repeat="note in navbar.notes.root.children | filter:query track by $index" ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'notebook_list_renderer.html'">
</li>
</div>
</ul>
@ -65,7 +63,7 @@ limitations under the License.
<!--TODO(bzz): move to Typeahead https://angular-ui.github.io/bootstrap -->
<form role="search"
style="width: 300px; display: inline-block; margin: 0 10px"
ng-submit="search()">
ng-submit="search(searchTerm)">
<div class="input-group">
<input
type="text"

49
zeppelin-zengine/pom.xml
View file

@ -115,16 +115,34 @@
<version>1.5.2</version>
<exclusions>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
@ -221,6 +239,27 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-server</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>

14
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -338,6 +338,14 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
}
public String getS3KMSKeyID() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
}
public String getS3EncryptionMaterialsProviderClass() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
}
public String getInterpreterDir() {
return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
}
@ -471,6 +479,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.markdown.Markdown,"
+ "org.apache.zeppelin.angular.AngularInterpreter,"
+ "org.apache.zeppelin.shell.ShellInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
+ "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ "org.apache.zeppelin.hive.HiveInterpreter,"
+ "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ "org.apache.zeppelin.file.HDFSFileInterpreter,"
@ -501,6 +513,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -40,6 +40,7 @@ import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.search.SearchService;
import com.google.gson.Gson;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -366,11 +367,15 @@ public class Note implements Serializable, ParagraphJobListener {
* Run all paragraphs sequentially.
*/
public void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (!p.isEnabled()) {
continue;
}
AuthenticationInfo authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUser(cronExecutingUser);
p.setAuthenticationInfo(authenticationInfo);
p.setNoteReplLoader(replLoader);
p.setListener(this);
Interpreter intp = replLoader.get(p.getRequiredReplName());

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -149,7 +149,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
int scriptHeadIndex = 0;
for (int i = 0; i < text.length(); i++) {
char ch = text.charAt(i);
if (ch == ' ' || ch == '\n' || ch == '(') {
if (Character.isWhitespace(ch) || ch == '(') {
scriptHeadIndex = i;
break;
}

169
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -26,6 +26,11 @@ import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
@ -37,7 +42,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
@ -70,69 +74,90 @@ public class S3NotebookRepo implements NotebookRepo {
// 3. Credential profiles file at the default location (~/.aws/credentials)
// shared by all AWS SDKs and the AWS CLI
// 4. Instance profile credentials delivered through the Amazon EC2 metadata service
private AmazonS3 s3client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
private static String bucketName = "";
private static String endpoint = "";
private String user = "";
private ZeppelinConfiguration conf;
private final AmazonS3 s3client;
private final String bucketName;
private final String user;
private final ZeppelinConfiguration conf;
public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
this.conf = conf;
bucketName = conf.getBucketName();
endpoint = conf.getEndpoint();
user = conf.getUser();
s3client.setEndpoint(endpoint);
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class
EncryptionMaterialsProvider emp = createCustomProvider(conf);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
else {
// regular S3
this.s3client = new AmazonS3Client(credentialsProvider);
}
// set S3 endpoint to use
s3client.setEndpoint(conf.getEndpoint());
}
/**
* Create an instance of a custom encryption materials provider class
* which supplies encryption keys to use when reading/writing data in S3.
*/
private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration conf)
throws IOException {
// use a custom encryption materials provider class
String empClassname = conf.getS3EncryptionMaterialsProviderClass();
EncryptionMaterialsProvider emp;
try {
Object empInstance = Class.forName(empClassname).newInstance();
if (empInstance instanceof EncryptionMaterialsProvider) {
emp = (EncryptionMaterialsProvider) empInstance;
}
else {
throw new IOException("Class " + empClassname + " does not implement "
+ EncryptionMaterialsProvider.class.getName());
}
}
catch (Exception e) {
throw new IOException("Unable to instantiate encryption materials provider class "
+ empClassname + ": " + e, e);
}
return emp;
}
@Override
public List<NoteInfo> list() throws IOException {
List<NoteInfo> infos = new LinkedList<NoteInfo>();
NoteInfo info = null;
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix(user + "/" + "notebook");
.withBucketName(bucketName)
.withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary :
objectListing.getObjectSummaries()) {
if (objectSummary.getKey().contains("note.json")) {
try {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
} catch (AmazonServiceException ase) {
LOG.warn("Caught an AmazonServiceException for some reason.\n" +
"Error Message: {}", ase.getMessage());
} catch (AmazonClientException ace) {
LOG.info("Caught an AmazonClientException, " +
"which means the client encountered " +
"an internal error while trying to communicate" +
" with S3, " +
"such as not being able to access the network.");
LOG.info("Error Message: " + ace.getMessage());
} catch (Exception e) {
LOG.error("Can't read note ", e);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonServiceException ase) {
LOG.warn("Caught an AmazonServiceException for some reason.\n" +
"Error Message: {}", ase.getMessage());
} catch (AmazonClientException ace) {
LOG.info("Caught an AmazonClientException, " +
"which means the client encountered " +
"an internal error while trying to communicate" +
" with S3, " +
"such as not being able to access the network.");
LOG.info("Error Message: " + ace.getMessage());
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
@ -142,19 +167,26 @@ public class S3NotebookRepo implements NotebookRepo {
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
S3Object s3object = s3client.getObject(new GetObjectRequest(
bucketName, key));
S3Object s3object;
try {
s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
}
catch (AmazonClientException ace) {
throw new IOException("Unable to retrieve object from S3: " + ace, ace);
}
InputStream ins = s3object.getObjectContent();
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
ins.close();
Note note = gson.fromJson(json, Note.class);
Note note;
try (InputStream ins = s3object.getObjectContent()) {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
note = gson.fromJson(json, Note.class);
}
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
p.setStatus(Status.ABORT);
}
}
return note;
}
@ -177,12 +209,18 @@ public class S3NotebookRepo implements NotebookRepo {
String key = user + "/" + "notebook" + "/" + note.id() + "/" + "note.json";
File file = File.createTempFile("note", "json");
file.deleteOnExit();
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
s3client.putObject(new PutObjectRequest(bucketName, key, file));
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
s3client.putObject(new PutObjectRequest(bucketName, key, file));
}
catch (AmazonClientException ace) {
throw new IOException("Unable to store note in S3: " + ace, ace);
}
finally {
FileUtils.deleteQuietly(file);
}
}
@Override
@ -191,13 +229,18 @@ public class S3NotebookRepo implements NotebookRepo {
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName).withPrefix(key);
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
@Override

195
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java Normal file
View file

@ -0,0 +1,195 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
/**
* ZeppelinHub repo class.
*/
public class ZeppelinHubRepo implements NotebookRepo {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinhubRestApiHandler.class);
private static final String DEFAULT_SERVER = "https://www.zeppelinhub.com";
static final String ZEPPELIN_CONF_PROP_NAME_SERVER = "zeppelinhub.api.address";
static final String ZEPPELIN_CONF_PROP_NAME_TOKEN = "zeppelinhub.api.token";
public static final String TOKEN_HEADER = "X-Zeppelin-Token";
private static final Gson GSON = new Gson();
private static final Note EMPTY_NOTE = new Note();
private final Client websocketClient;
private String token;
private ZeppelinhubRestApiHandler restApiClient;
public ZeppelinHubRepo(ZeppelinConfiguration conf) {
String zeppelinHubUrl = getZeppelinHubUrl(conf);
LOG.info("Initializing ZeppelinHub integration module");
token = conf.getString("ZEPPELINHUB_API_TOKEN", ZEPPELIN_CONF_PROP_NAME_TOKEN, "");
restApiClient = ZeppelinhubRestApiHandler.newInstance(zeppelinHubUrl, token);
websocketClient = Client.initialize(getZeppelinWebsocketUri(conf),
getZeppelinhubWebsocketUri(conf), token, conf);
websocketClient.start();
}
private String getZeppelinHubWsUri(URI api) throws URISyntaxException {
URI apiRoot = api;
String scheme = apiRoot.getScheme();
int port = apiRoot.getPort();
if (port <= 0) {
port = (scheme != null && scheme.equals("https")) ? 443 : 80;
}
if (scheme == null) {
LOG.info("{} is not a valid zeppelinhub server address. proceed with default address {}",
apiRoot, DEFAULT_SERVER);
apiRoot = new URI(DEFAULT_SERVER);
scheme = apiRoot.getScheme();
port = apiRoot.getPort();
if (port <= 0) {
port = (scheme != null && scheme.equals("https")) ? 443 : 80;
}
}
String ws = scheme.equals("https") ? "wss://" : "ws://";
return ws + apiRoot.getHost() + ":" + port + "/async";
}
String getZeppelinhubWebsocketUri(ZeppelinConfiguration conf) {
String zeppelinHubUri = StringUtils.EMPTY;
try {
zeppelinHubUri = getZeppelinHubWsUri(new URI(conf.getString("ZEPPELINHUB_API_ADDRESS",
ZEPPELIN_CONF_PROP_NAME_SERVER, DEFAULT_SERVER)));
} catch (URISyntaxException e) {
LOG.error("Cannot get ZeppelinHub URI", e);
}
return zeppelinHubUri;
}
private String getZeppelinWebsocketUri(ZeppelinConfiguration conf) {
int port = conf.getServerPort();
if (port <= 0) {
port = 80;
}
String ws = conf.useSsl() ? "wss" : "ws";
return ws + "://localhost:" + port + "/ws";
}
// Used in tests
void setZeppelinhubRestApiHandler(ZeppelinhubRestApiHandler zeppelinhub) {
restApiClient = zeppelinhub;
}
String getZeppelinHubUrl(ZeppelinConfiguration conf) {
if (conf == null) {
LOG.error("Invalid configuration, cannot be null. Using default address {}", DEFAULT_SERVER);
return DEFAULT_SERVER;
}
URI apiRoot;
String zeppelinhubUrl;
try {
String url = conf.getString("ZEPPELINHUB_API_ADDRESS",
ZEPPELIN_CONF_PROP_NAME_SERVER,
DEFAULT_SERVER);
apiRoot = new URI(url);
} catch (URISyntaxException e) {
LOG.error("Invalid zeppelinhub url, using default address {}", DEFAULT_SERVER, e);
return DEFAULT_SERVER;
}
String scheme = apiRoot.getScheme();
if (scheme == null) {
LOG.info("{} is not a valid zeppelinhub server address. proceed with default address {}",
apiRoot, DEFAULT_SERVER);
zeppelinhubUrl = DEFAULT_SERVER;
} else {
zeppelinhubUrl = scheme + "://" + apiRoot.getHost();
if (apiRoot.getPort() > 0) {
zeppelinhubUrl += ":" + apiRoot.getPort();
}
}
return zeppelinhubUrl;
}
@Override
public List<NoteInfo> list() throws IOException {
String response = restApiClient.asyncGet("");
List<NoteInfo> notes = GSON.fromJson(response, new TypeToken<List<NoteInfo>>() {}.getType());
if (notes == null) {
return Collections.emptyList();
}
LOG.info("ZeppelinHub REST API listing notes ");
return notes;
}
@Override
public Note get(String noteId) throws IOException {
if (StringUtils.isBlank(noteId)) {
return EMPTY_NOTE;
}
//String response = zeppelinhubHandler.get(noteId);
String response = restApiClient.asyncGet(noteId);
Note note = GSON.fromJson(response, Note.class);
if (note == null) {
return EMPTY_NOTE;
}
LOG.info("ZeppelinHub REST API get note {} ", noteId);
return note;
}
@Override
public void save(Note note) throws IOException {
if (note == null) {
throw new IOException("Zeppelinhub failed to save empty note");
}
String notebook = GSON.toJson(note);
restApiClient.asyncPut(notebook);
LOG.info("ZeppelinHub REST API saving note {} ", note.id());
}
@Override
public void remove(String noteId) throws IOException {
restApiClient.asyncDel(noteId);
LOG.info("ZeppelinHub REST API removing note {} ", noteId);
}
@Override
public void close() {
websocketClient.stop();
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
}
}

210
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/rest/ZeppelinhubRestApiHandler.java Normal file
View file

@ -0,0 +1,210 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.rest;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.client.HttpClient;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.api.Result;
import org.eclipse.jetty.client.util.BufferingResponseListener;
import org.eclipse.jetty.client.util.InputStreamResponseListener;
import org.eclipse.jetty.client.util.StringContentProvider;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* REST API handler.
*
*/
public class ZeppelinhubRestApiHandler {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinhubRestApiHandler.class);
public static final String ZEPPELIN_TOKEN_HEADER = "X-Zeppelin-Token";
private static final String DEFAULT_API_PATH = "/api/v1/zeppelin";
private static boolean PROXY_ON = false;
private static String PROXY_HOST;
private static int PROXY_PORT;
private final HttpClient client;
private final String zepelinhubUrl;
private final String token;
public static ZeppelinhubRestApiHandler newInstance(String zeppelinhubUrl,
String token) {
return new ZeppelinhubRestApiHandler(zeppelinhubUrl, token);
}
private ZeppelinhubRestApiHandler(String zeppelinhubUrl, String token) {
this.zepelinhubUrl = zeppelinhubUrl + DEFAULT_API_PATH + "/";
this.token = token;
//TODO(khalid):to make proxy conf consistent with Zeppelin confs
//readProxyConf();
client = getAsyncClient();
try {
client.start();
} catch (Exception e) {
LOG.error("Cannot initialize ZeppelinHub REST async client", e);
}
}
private void readProxyConf() {
//try reading http_proxy
String proxyHostString = StringUtils.isBlank(System.getenv("http_proxy")) ?
System.getenv("HTTP_PROXY") : System.getenv("http_proxy");
if (StringUtils.isBlank(proxyHostString)) {
//try https_proxy if no http_proxy
proxyHostString = StringUtils.isBlank(System.getenv("https_proxy")) ?
System.getenv("HTTPS_PROXY") : System.getenv("https_proxy");
}
if (StringUtils.isBlank(proxyHostString)) {
PROXY_ON = false;
} else {
// host format - http://domain:port/
String[] parts = proxyHostString.replaceAll("/", "").split(":");
if (parts.length != 3) {
LOG.warn("Proxy host format is incorrect {}, e.g. http://domain:port/", proxyHostString);
PROXY_ON = false;
return;
}
PROXY_HOST = parts[1];
PROXY_PORT = Integer.parseInt(parts[2]);
LOG.info("Proxy protocol: {}, domain: {}, port: {}", parts[0], parts[1], parts[2]);
PROXY_ON = true;
}
}
private HttpClient getAsyncClient() {
SslContextFactory sslContextFactory = new SslContextFactory();
HttpClient httpClient = new HttpClient(sslContextFactory);
// Configure HttpClient
httpClient.setFollowRedirects(false);
httpClient.setMaxConnectionsPerDestination(100);
// Config considerations
//TODO(khalid): consider using proxy
//TODO(khalid): consider whether require to follow redirects
//TODO(khalid): consider multi-threaded connection manager case
return httpClient;
}
public String asyncGet(String argument) throws IOException {
String note = StringUtils.EMPTY;
InputStreamResponseListener listener = new InputStreamResponseListener();
client.newRequest(zepelinhubUrl + argument)
.header(ZEPPELIN_TOKEN_HEADER, token)
.send(listener);
// Wait for the response headers to arrive
Response response;
try {
response = listener.get(30, TimeUnit.SECONDS);
} catch (InterruptedException | TimeoutException | ExecutionException e) {
LOG.error("Cannot perform Get request to ZeppelinHub", e);
throw new IOException("Cannot load note from ZeppelinHub", e);
}
int code = response.getStatus();
if (code == 200) {
try (InputStream responseContent = listener.getInputStream()) {
note = IOUtils.toString(responseContent, "UTF-8");
}
} else {
LOG.error("ZeppelinHub Get {} returned with status {} ", zepelinhubUrl + argument, code);
throw new IOException("Cannot load note from ZeppelinHub");
}
return note;
}
public void asyncPut(String jsonNote) throws IOException {
if (StringUtils.isBlank(jsonNote)) {
LOG.error("Cannot save empty note/string to ZeppelinHub");
return;
}
client.newRequest(zepelinhubUrl).method(HttpMethod.PUT)
.header(ZEPPELIN_TOKEN_HEADER, token)
.content(new StringContentProvider(jsonNote, "UTF-8"), "application/json;charset=UTF-8")
.send(new BufferingResponseListener() {
@Override
public void onComplete(Result res) {
if (!res.isFailed() && res.getResponse().getStatus() == 200) {
LOG.info("Successfully saved note to ZeppelinHub with {}",
res.getResponse().getStatus());
} else {
LOG.warn("Failed to save note to ZeppelinHub with HttpStatus {}",
res.getResponse().getStatus());
}
}
@Override
public void onFailure(Response response, Throwable failure) {
LOG.error("Failed to save note to ZeppelinHub: {}", response.getReason(), failure);
}
});
}
public void asyncDel(String argument) {
if (StringUtils.isBlank(argument)) {
LOG.error("Cannot delete empty note from ZeppelinHub");
return;
}
client.newRequest(zepelinhubUrl + argument)
.method(HttpMethod.DELETE)
.header(ZEPPELIN_TOKEN_HEADER, token)
.send(new BufferingResponseListener() {
@Override
public void onComplete(Result res) {
if (!res.isFailed() && res.getResponse().getStatus() == 200) {
LOG.info("Successfully removed note from ZeppelinHub with {}",
res.getResponse().getStatus());
} else {
LOG.warn("Failed to remove note from ZeppelinHub with HttpStatus {}",
res.getResponse().getStatus());
}
}
@Override
public void onFailure(Response response, Throwable failure) {
LOG.error("Failed to remove note from ZeppelinHub: {}", response.getReason(), failure);
}
});
}
public void close() {
try {
client.stop();
} catch (Exception e) {
LOG.info("Couldn't stop ZeppelinHub client properly", e);
}
}
}

228
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java Normal file
View file

@ -0,0 +1,228 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.security;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.security.Key;
import java.util.Collections;
import java.util.Map;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
/**
* Authentication module.
*
*/
public class Authentication implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(Authentication.class);
private String principal = "anonymous";
private String ticket = "anonymous";
private String roles = StringUtils.EMPTY;
private final HttpClient client;
private String loginEndpoint;
// Cipher is an AES in CBC mode
private static final String CIPHER_ALGORITHM = "AES";
private static final String CIPHER_MODE = "AES/CBC/PKCS5PADDING";
private static final String KEY = "AbtEr99DxsWWbJkP";
private static final int ivSize = 16;
private static final String ZEPPELIN_CONF_ANONYMOUS_ALLOWED = "zeppelin.anonymous.allowed";
private static final String ZEPPELINHUB_USER_KEY = "zeppelinhub.user.key";
private String token;
private boolean authEnabled;
private boolean authenticated;
String userKey;
private Gson gson = new Gson();
private static Authentication instance = null;
public static Authentication initialize(String token, ZeppelinConfiguration conf) {
if (instance == null && conf != null) {
instance = new Authentication(token, conf);
}
return instance;
}
public static Authentication getInstance() {
return instance;
}
private Authentication(String token, ZeppelinConfiguration conf) {
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
client = new HttpClient(connectionManager);
this.token = token;
authEnabled = !conf.getBoolean("ZEPPELIN_ALLOW_ANONYMOUS",
ZEPPELIN_CONF_ANONYMOUS_ALLOWED, true);
userKey = conf.getString("ZEPPELINHUB_USER_KEY",
ZEPPELINHUB_USER_KEY, "");
loginEndpoint = getLoginEndpoint(conf);
}
public String getPrincipal() {
return this.principal;
}
public String getTicket() {
return this.ticket;
}
public String getRoles() {
return this.roles;
}
public boolean isAuthenticated() {
return authenticated;
}
private String getLoginEndpoint(ZeppelinConfiguration conf) {
int port = conf.getInt("ZEPPELIN_PORT", "zeppelin.server.port" , 8080);
if (port <= 0) {
port = 8080;
}
String scheme = "http";
if (conf.useSsl()) {
scheme = "https";
}
String endpoint = scheme + "://localhost:" + port + "/api/login";
return endpoint;
}
public boolean authenticate() {
if (authEnabled) {
if (!StringUtils.isEmpty(userKey)) {
String authKey = getAuthKey(userKey);
Map<String, String> authCredentials = login(authKey, loginEndpoint);
if (isEmptyMap(authCredentials)) {
return false;
}
principal = authCredentials.containsKey("principal") ? authCredentials.get("principal")
: principal;
ticket = authCredentials.containsKey("ticket") ? authCredentials.get("ticket") : ticket;
roles = authCredentials.containsKey("roles") ? authCredentials.get("roles") : roles;
LOG.info("Authenticated into Zeppelin as {} and roles {}", principal, roles);
return true;
} else {
LOG.warn("ZEPPELINHUB_USER_KEY isn't provided. Please provide your credentials"
+ "for your instance in ZeppelinHub website and generate your key.");
}
}
return false;
}
// returns login:password
private String getAuthKey(String userKey) {
LOG.debug("Encrypted user key is {}", userKey);
if (StringUtils.isBlank(userKey)) {
LOG.warn("ZEPPELINHUB_USER_KEY is blank");
return StringUtils.EMPTY;
}
//use hashed token as a salt
String hashedToken = Integer.toString(token.hashCode());
return decrypt(userKey, hashedToken);
}
private String decrypt(String value, String initVector) {
LOG.debug("IV is {}, IV length is {}", initVector, initVector.length());
if (StringUtils.isBlank(value) || StringUtils.isBlank(initVector)) {
LOG.error("String to decode or salt is not provided");
return StringUtils.EMPTY;
}
try {
IvParameterSpec iv = generateIV(initVector);
Key key = generateKey();
Cipher cipher = Cipher.getInstance(CIPHER_MODE);
cipher.init(Cipher.DECRYPT_MODE, key, iv);
byte[] decryptedString = Base64.decodeBase64(toBytes(value));
decryptedString = cipher.doFinal(decryptedString);
return new String(decryptedString);
} catch (GeneralSecurityException e) {
LOG.error("Error when decrypting", e);
return StringUtils.EMPTY;
}
}
@SuppressWarnings("unchecked")
private Map<String, String> login(String authKey, String endpoint) {
String[] credentials = authKey.split(":");
if (credentials.length != 2) {
return Collections.emptyMap();
}
PostMethod post = new PostMethod(endpoint);
post.addRequestHeader("Origin", "http://localhost");
post.addParameter(new NameValuePair("userName", credentials[0]));
post.addParameter(new NameValuePair("password", credentials[1]));
try {
int code = client.executeMethod(post);
if (code == HttpStatus.SC_OK) {
String content = post.getResponseBodyAsString();
Map<String, Object> resp = gson.fromJson(content,
new TypeToken<Map<String, Object>>() {}.getType());
LOG.info("Received from Zeppelin LoginRestApi : " + content);
return (Map<String, String>) resp.get("body");
} else {
LOG.error("Failed Zeppelin login {}, status code {}", endpoint, code);
return Collections.emptyMap();
}
} catch (IOException e) {
LOG.error("Cannot login into Zeppelin", e);
return Collections.emptyMap();
}
}
private Key generateKey() {
return new SecretKeySpec(toBytes(KEY), CIPHER_ALGORITHM);
}
private byte[] toBytes(String value) {
byte[] bytes;
try {
bytes = value.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
LOG.warn("UTF-8 isn't supported ", e);
bytes = value.getBytes();
}
return bytes;
}
private IvParameterSpec generateIV(String ivString) {
byte[] ivFromBytes = toBytes(ivString);
byte[] iv16ToBytes = new byte[ivSize];
System.arraycopy(ivFromBytes, 0, iv16ToBytes, 0, Math.min(ivFromBytes.length, ivSize));
return new IvParameterSpec(iv16ToBytes);
}
private boolean isEmptyMap(Map<String, String> map) {
return map == null || map.isEmpty();
}
@Override
public void run() {
authenticated = authenticate();
LOG.info("Scheduled authentication status is {}", authenticated);
}
}

79
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/Client.java Normal file
View file

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.socket.Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Client to connect Zeppelin and ZeppelinHub via websocket API.
* Implemented using singleton pattern.
*
*/
public class Client {
private static final Logger LOG = LoggerFactory.getLogger(Client.class);
private final ZeppelinhubClient zeppelinhubClient;
private final ZeppelinClient zeppelinClient;
private static Client instance = null;
public static Client initialize(String zeppelinUri, String zeppelinhubUri, String token,
ZeppelinConfiguration conf) {
if (instance == null) {
instance = new Client(zeppelinUri, zeppelinhubUri, token, conf);
}
return instance;
}
public static Client getInstance() {
return instance;
}
private Client(String zeppelinUri, String zeppelinhubUri, String token,
ZeppelinConfiguration conf) {
LOG.debug("Init Client");
zeppelinhubClient = ZeppelinhubClient.initialize(zeppelinhubUri, token);
zeppelinClient = ZeppelinClient.initialize(zeppelinUri, token, conf);
}
public void start() {
if (zeppelinhubClient != null) {
zeppelinhubClient.start();
}
if (zeppelinClient != null) {
zeppelinClient.start();
}
}
public void stop() {
if (zeppelinhubClient != null) {
zeppelinhubClient.stop();
}
if (zeppelinClient != null) {
zeppelinClient.stop();
}
}
public void relayToZeppelinHub(String message) {
zeppelinhubClient.send(message);
}
public void relayToZeppelin(Message message, String noteId) {
zeppelinClient.send(message, noteId);
}
}

286
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClient.java Normal file
View file

@ -0,0 +1,286 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.repo.zeppelinhub.security.Authentication;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.ZeppelinWebsocket;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinhubMessage;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler.SchedulerService;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler.ZeppelinHeartbeat;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler.ZeppelinHubHeartbeat;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.client.ClientUpgradeRequest;
import org.eclipse.jetty.websocket.client.WebSocketClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
/**
* Zeppelin websocket client.
*
*/
public class ZeppelinClient {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinClient.class);
private final URI zeppelinWebsocketUrl;
private final String zeppelinhubToken;
private final WebSocketClient wsClient;
private static Gson gson;
private ConcurrentHashMap<String, Session> zeppelinConnectionMap;
private static ZeppelinClient instance = null;
private SchedulerService schedulerService;
private Authentication authModule;
private static final int min = 60;
public static ZeppelinClient initialize(String zeppelinUrl, String token,
ZeppelinConfiguration conf) {
if (instance == null) {
instance = new ZeppelinClient(zeppelinUrl, token, conf);
}
return instance;
}
public static ZeppelinClient getInstance() {
return instance;
}
private ZeppelinClient(String zeppelinUrl, String token, ZeppelinConfiguration conf) {
zeppelinWebsocketUrl = URI.create(zeppelinUrl);
zeppelinhubToken = token;
wsClient = createNewWebsocketClient();
gson = new Gson();
zeppelinConnectionMap = new ConcurrentHashMap<>();
schedulerService = SchedulerService.getInstance();
authModule = Authentication.initialize(token, conf);
if (authModule != null) {
SchedulerService.getInstance().addOnce(authModule, 10);
}
LOG.info("Initialized Zeppelin websocket client on {}", zeppelinWebsocketUrl);
}
private WebSocketClient createNewWebsocketClient() {
SslContextFactory sslContextFactory = new SslContextFactory();
WebSocketClient client = new WebSocketClient(sslContextFactory);
client.setMaxIdleTimeout(5 * min * 1000);
//TODO(khalid): other client settings
return client;
}
public void start() {
try {
if (wsClient != null) {
wsClient.start();
addRoutines();
} else {
LOG.warn("Cannot start zeppelin websocket client - isn't initialized");
}
} catch (Exception e) {
LOG.error("Cannot start Zeppelin websocket client", e);
}
}
private void addRoutines() {
schedulerService.add(ZeppelinHeartbeat.newInstance(this), 15, 4 * min);
}
public void stop() {
try {
if (wsClient != null) {
removeAllZeppelinConnections();
wsClient.stop();
} else {
LOG.warn("Cannot stop zeppelin websocket client - isn't initialized");
}
} catch (Exception e) {
LOG.error("Cannot stop Zeppelin websocket client", e);
}
}
public String serialize(Message zeppelinMsg) {
if (credentialsAvailable()) {
zeppelinMsg.principal = authModule.getPrincipal();
zeppelinMsg.ticket = authModule.getTicket();
zeppelinMsg.roles = authModule.getRoles();
}
String msg = gson.toJson(zeppelinMsg);
return msg;
}
private boolean credentialsAvailable() {
return Authentication.getInstance() != null && Authentication.getInstance().isAuthenticated();
}
public Message deserialize(String zeppelinMessage) {
if (StringUtils.isBlank(zeppelinMessage)) {
return null;
}
Message msg;
try {
msg = gson.fromJson(zeppelinMessage, Message.class);
} catch (JsonSyntaxException ex) {
LOG.error("Cannot deserialize zeppelin message", ex);
msg = null;
}
return msg;
}
public void send(Message msg, String noteId) {
Session noteSession = getZeppelinConnection(noteId);
if (!isSessionOpen(noteSession)) {
LOG.error("Cannot open websocket connection to Zeppelin note {}", noteId);
return;
}
noteSession.getRemote().sendStringByFuture(serialize(msg));
}
private boolean isSessionOpen(Session session) {
return (session != null) && (session.isOpen());
}
/* per notebook based ws connection, returns null if can't connect */
public Session getZeppelinConnection(String noteId) {
if (StringUtils.isBlank(noteId)) {
LOG.warn("Cannot return websocket connection for blank noteId");
return null;
}
if (zeppelinConnectionMap.containsKey(noteId)) {
LOG.info("Connection for {} exists in map", noteId);
return getNoteSession(noteId);
}
//TODO(khalid): clean log later
LOG.info("Creating Zeppelin websocket connection {} {}", zeppelinWebsocketUrl, noteId);
return openNoteSession(noteId);
}
private Message zeppelinGetNoteMsg(String noteId) {
Message getNoteMsg = new Message(Message.OP.GET_NOTE);
HashMap<String, Object> data = new HashMap<String, Object>();
data.put("id", noteId);
getNoteMsg.data = data;
return getNoteMsg;
}
private Session getNoteSession(String noteId) {
Session session = zeppelinConnectionMap.get(noteId);
if (session == null || !session.isOpen()) {
LOG.info("Not connection to {}", noteId);
zeppelinConnectionMap.remove(noteId);
session = openNoteSession(noteId);
}
return session;
}
private Session openNoteSession(String noteId) {
ClientUpgradeRequest request = new ClientUpgradeRequest();
ZeppelinWebsocket socket = new ZeppelinWebsocket(noteId);
Future<Session> future = null;
Session session = null;
try {
future = wsClient.connect(socket, zeppelinWebsocketUrl, request);
session = future.get();
} catch (IOException | InterruptedException | ExecutionException e) {
LOG.error("Couldn't establish websocket connection to Zeppelin ", e);
return session;
}
if (zeppelinConnectionMap.containsKey(noteId)) {
session.close();
session = zeppelinConnectionMap.get(noteId);
} else {
String getNote = serialize(zeppelinGetNoteMsg(noteId));
// TODO(khalid): may need to check return whether successful
session.getRemote().sendStringByFuture(getNote);
zeppelinConnectionMap.put(noteId, session);
}
return session;
}
public void handleMsgFromZeppelin(String message, String noteId) {
Map<String, String> meta = new HashMap<String, String>();
meta.put("token", zeppelinhubToken);
meta.put("noteId", noteId);
Message zeppelinMsg = deserialize(message);
if (zeppelinMsg == null) {
return;
}
ZeppelinhubMessage hubMsg = ZeppelinhubMessage.newMessage(zeppelinMsg, meta);
Client client = Client.getInstance();
if (client == null) {
LOG.warn("Client isn't initialized yet");
return;
}
client.relayToZeppelinHub(hubMsg.serialize());
}
/**
* Close and remove ZeppelinConnection
*/
public void removeZeppelinConnection(String noteId) {
if (zeppelinConnectionMap.containsKey(noteId)) {
Session conn = zeppelinConnectionMap.get(noteId);
if (conn.isOpen()) {
conn.close();
}
zeppelinConnectionMap.remove(noteId);
}
// TODO(khalid): clean log later
LOG.info("Removed Zeppelin ws connection for the following note {}", noteId);
}
/**
* Close and remove all ZeppelinConnection
*/
public void removeAllZeppelinConnections() {
for (Map.Entry<String, Session> entry: zeppelinConnectionMap.entrySet()) {
if (isSessionOpen(entry.getValue())) {
entry.getValue().close();
}
zeppelinConnectionMap.remove(entry.getKey());
}
LOG.info("Removed all Zeppelin ws connections");
}
public void pingAllNotes() {
for (Map.Entry<String, Session> entry: zeppelinConnectionMap.entrySet()) {
if (isSessionOpen(entry.getValue())) {
send(new Message(OP.PING), entry.getKey());
} else {
// for cleanup
zeppelinConnectionMap.remove(entry.getKey());
}
}
}
public int countConnectedNotes() {
return zeppelinConnectionMap.size();
}
}

251
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClient.java Normal file
View file

@ -0,0 +1,251 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
import java.io.IOException;
import java.net.HttpCookie;
import java.net.URI;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener.ZeppelinhubWebsocket;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinHubOp;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinhubMessage;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler.SchedulerService;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler.ZeppelinHubHeartbeat;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.session.ZeppelinhubSession;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.client.ClientUpgradeRequest;
import org.eclipse.jetty.websocket.client.WebSocketClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.util.json.JSONArray;
import com.amazonaws.util.json.JSONException;
import com.amazonaws.util.json.JSONObject;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
/**
* Manage a zeppelinhub websocket connection.
*/
public class ZeppelinhubClient {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinhubClient.class);
private final WebSocketClient client;
private final URI zeppelinhubWebsocketUrl;
private final ClientUpgradeRequest conectionRequest;
private final String zeppelinhubToken;
private static final int MB = 1048576;
private static final int MAXIMUN_TEXT_SIZE = 64 * MB;
private static final long CONNECTION_IDLE_TIME = TimeUnit.SECONDS.toMillis(30);
private static ZeppelinhubClient instance = null;
private static Gson gson;
private SchedulerService schedulerService;
private ZeppelinhubSession zeppelinhubSession;
public static ZeppelinhubClient initialize(String zeppelinhubUrl, String token) {
if (instance == null) {
instance = new ZeppelinhubClient(zeppelinhubUrl, token);
}
return instance;
}
public static ZeppelinhubClient getInstance() {
return instance;
}
private ZeppelinhubClient(String url, String token) {
zeppelinhubWebsocketUrl = URI.create(url);
client = createNewWebsocketClient();
conectionRequest = setConnectionrequest(token);
zeppelinhubToken = token;
schedulerService = SchedulerService.create(10);
gson = new Gson();
LOG.info("Initialized ZeppelinHub websocket client on {}", zeppelinhubWebsocketUrl);
}
public void start() {
try {
client.start();
zeppelinhubSession = connect();
addRoutines();
} catch (Exception e) {
LOG.error("Cannot connect to zeppelinhub via websocket", e);
}
}
public void stop() {
LOG.info("Stopping Zeppelinhub websocket client");
try {
zeppelinhubSession.close();
schedulerService.close();
client.stop();
} catch (Exception e) {
LOG.error("Cannot stop zeppelinhub websocket client", e);
}
}
public String getToken() {
return this.zeppelinhubToken;
}
public void send(String msg) {
if (!isConnectedToZeppelinhub()) {
LOG.info("Zeppelinhub connection is not open, opening it");
zeppelinhubSession = connect();
if (zeppelinhubSession == ZeppelinhubSession.EMPTY) {
LOG.warn("While connecting to ZeppelinHub received empty session, cannot send the message");
return;
}
}
zeppelinhubSession.sendByFuture(msg);
}
private boolean isConnectedToZeppelinhub() {
return (zeppelinhubSession != null && zeppelinhubSession.isSessionOpen());
}
private ZeppelinhubSession connect() {
ZeppelinhubSession zeppelinSession;
try {
ZeppelinhubWebsocket ws = ZeppelinhubWebsocket.newInstance(zeppelinhubToken);
Future<Session> future = client.connect(ws, zeppelinhubWebsocketUrl, conectionRequest);
Session session = future.get();
zeppelinSession = ZeppelinhubSession.createInstance(session, zeppelinhubToken);
} catch (IOException | InterruptedException | ExecutionException e) {
LOG.info("Couldnt connect to zeppelinhub", e);
zeppelinSession = ZeppelinhubSession.EMPTY;
}
return zeppelinSession;
}
private ClientUpgradeRequest setConnectionrequest(String token) {
ClientUpgradeRequest request = new ClientUpgradeRequest();
request.setCookies(Lists.newArrayList(new HttpCookie(ZeppelinHubRepo.TOKEN_HEADER, token)));
return request;
}
private WebSocketClient createNewWebsocketClient() {
SslContextFactory sslContextFactory = new SslContextFactory();
WebSocketClient client = new WebSocketClient(sslContextFactory);
client.setMaxTextMessageBufferSize(MAXIMUN_TEXT_SIZE);
client.setMaxIdleTimeout(CONNECTION_IDLE_TIME);
return client;
}
private void addRoutines() {
schedulerService.add(ZeppelinHubHeartbeat.newInstance(this), 10, 23);
}
public void handleMsgFromZeppelinHub(String message) {
ZeppelinhubMessage hubMsg = ZeppelinhubMessage.deserialize(message);
if (hubMsg.equals(ZeppelinhubMessage.EMPTY)) {
LOG.error("Cannot handle ZeppelinHub message is empty");
return;
}
String op = StringUtils.EMPTY;
if (hubMsg.op instanceof String) {
op = (String) hubMsg.op;
} else {
LOG.error("Message OP from ZeppelinHub isn't string {}", hubMsg.op);
return;
}
if (ZeppelinhubUtils.isZeppelinHubOp(op)) {
handleZeppelinHubOpMsg(ZeppelinhubUtils.toZeppelinHubOp(op), hubMsg, message);
} else if (ZeppelinhubUtils.isZeppelinOp(op)) {
forwardToZeppelin(ZeppelinhubUtils.toZeppelinOp(op), hubMsg);
}
}
private void handleZeppelinHubOpMsg(ZeppelinHubOp op, ZeppelinhubMessage hubMsg, String msg) {
if (op == null || msg.equals(ZeppelinhubMessage.EMPTY)) {
LOG.error("Cannot handle empty op or msg");
return;
}
switch (op) {
case RUN_NOTEBOOK:
runAllParagraph(hubMsg.meta.get("noteId"), msg);
break;
default:
LOG.warn("Received {} from ZeppelinHub, not handled", op);
break;
}
}
@SuppressWarnings("unchecked")
private void forwardToZeppelin(Message.OP op, ZeppelinhubMessage hubMsg) {
Message zeppelinMsg = new Message(op);
if (!(hubMsg.data instanceof Map)) {
LOG.error("Data field of message from ZeppelinHub isn't in correct Map format");
return;
}
zeppelinMsg.data = (Map<String, Object>) hubMsg.data;
Client client = Client.getInstance();
if (client == null) {
LOG.warn("Base client isn't initialized, returning");
return;
}
client.relayToZeppelin(zeppelinMsg, hubMsg.meta.get("noteId"));
}
boolean runAllParagraph(String noteId, String hubMsg) {
LOG.info("Running paragraph with noteId {}", noteId);
try {
JSONObject data = new JSONObject(hubMsg);
if (data.equals(JSONObject.NULL) || !(data.get("data") instanceof JSONArray)) {
LOG.error("Wrong \"data\" format for RUN_NOTEBOOK");
return false;
}
Client client = Client.getInstance();
if (client == null) {
LOG.warn("Base client isn't initialized, returning");
return false;
}
Message zeppelinMsg = new Message(OP.RUN_PARAGRAPH);
JSONArray paragraphs = data.getJSONArray("data");
for (int i = 0; i < paragraphs.length(); i++) {
if (!(paragraphs.get(i) instanceof JSONObject)) {
LOG.warn("Wrong \"paragraph\" format for RUN_NOTEBOOK");
continue;
}
zeppelinMsg.data = gson.fromJson(paragraphs.getString(i),
new TypeToken<Map<String, Object>>(){}.getType());
client.relayToZeppelin(zeppelinMsg, noteId);
LOG.info("\nSending RUN_PARAGRAPH message to Zeppelin ");
}
} catch (JSONException e) {
LOG.error("Failed to parse RUN_NOTEBOOK message from ZeppelinHub ", e);
return false;
}
return true;
}
}

74
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/listener/ZeppelinWebsocket.java Normal file
View file

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinClient;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Zeppelin websocket listener class.
*
*/
public class ZeppelinWebsocket implements WebSocketListener {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinWebsocket.class);
public Session connection;
public String noteId;
public ZeppelinWebsocket(String noteId) {
this.noteId = noteId;
}
@Override
public void onWebSocketBinary(byte[] arg0, int arg1, int arg2) {
}
@Override
public void onWebSocketClose(int code, String message) {
LOG.info("Zeppelin connection closed with code: {}, message: {}", code, message);
// parentClient.removeConnMap(noteId);
}
@Override
public void onWebSocketConnect(Session session) {
LOG.info("Zeppelin connection opened");
this.connection = session;
}
@Override
public void onWebSocketError(Throwable e) {
LOG.warn("Zeppelin socket connection error ", e);
}
@Override
public void onWebSocketText(String data) {
LOG.debug("Zeppelin client received Message: " + data);
// propagate to ZeppelinHub
try {
ZeppelinClient zeppelinClient = ZeppelinClient.getInstance();
if (zeppelinClient != null) {
zeppelinClient.handleMsgFromZeppelin(data, noteId);
}
} catch (Exception e) {
LOG.error("Failed to send message to ZeppelinHub: ", e);
}
}
}

82
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/listener/ZeppelinhubWebsocket.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.listener;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Zeppelinhub websocket handler.
*/
public class ZeppelinhubWebsocket implements WebSocketListener {
private Logger LOG = LoggerFactory.getLogger(ZeppelinhubWebsocket.class);
private Session zeppelinHubSession;
private final String token;
private ZeppelinhubWebsocket(String token) {
this.token = token;
}
public static ZeppelinhubWebsocket newInstance(String token) {
return new ZeppelinhubWebsocket(token);
}
@Override
public void onWebSocketBinary(byte[] payload, int offset, int len) {}
@Override
public void onWebSocketClose(int statusCode, String reason) {
LOG.info("Closing websocket connection [{}] : {}", statusCode, reason);
send(ZeppelinhubUtils.deadMessage(token));
this.zeppelinHubSession = null;
}
@Override
public void onWebSocketConnect(Session session) {
LOG.info("Opening a new session to Zeppelinhub {}", session.hashCode());
this.zeppelinHubSession = session;
send(ZeppelinhubUtils.liveMessage(token));
}
@Override
public void onWebSocketError(Throwable cause) {
LOG.error("Got error", cause);
}
@Override
public void onWebSocketText(String message) {
// handle message from ZeppelinHub.
ZeppelinhubClient client = ZeppelinhubClient.getInstance();
if (client != null) {
client.handleMsgFromZeppelinHub(message);
}
}
private boolean isSessionOpen() {
return ((zeppelinHubSession != null) && (zeppelinHubSession.isOpen())) ? true : false;
}
private void send(String msg) {
if (isSessionOpen()) {
zeppelinHubSession.getRemote().sendStringByFuture(msg);
}
}
}

30
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinHubOp.java Normal file
View file

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol;
/**
* Zeppelinhub Op.
*/
public enum ZeppelinHubOp {
LIVE,
DEAD,
PING,
PONG,
RUN_NOTEBOOK,
WELCOME,
ZEPPELIN_STATUS
}

85
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessage.java Normal file
View file

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.Client;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
/**
* Zeppelinhub message class.
*
*/
public class ZeppelinhubMessage {
private static final Gson gson = new Gson();
private static final Logger LOG = LoggerFactory.getLogger(Client.class);
public static final ZeppelinhubMessage EMPTY = new ZeppelinhubMessage();
public Object op;
public Object data;
public Map<String, String> meta = Maps.newHashMap();
private ZeppelinhubMessage() {
this.op = OP.LIST_NOTES;
this.data = null;
}
private ZeppelinhubMessage(Object op, Object data, Map<String, String> meta) {
this.op = op;
this.data = data;
this.meta = meta;
}
public static ZeppelinhubMessage newMessage(Object op, Object data, Map<String, String> meta) {
return new ZeppelinhubMessage(op, data, meta);
}
public static ZeppelinhubMessage newMessage(Message zeppelinMsg, Map<String, String> meta) {
if (zeppelinMsg == null) {
return EMPTY;
}
return new ZeppelinhubMessage(zeppelinMsg.op, zeppelinMsg.data, meta);
}
public String serialize() {
return gson.toJson(this, ZeppelinhubMessage.class);
}
public static ZeppelinhubMessage deserialize(String zeppelinhubMessage) {
if (StringUtils.isBlank(zeppelinhubMessage)) {
return EMPTY;
}
ZeppelinhubMessage msg;
try {
msg = gson.fromJson(zeppelinhubMessage, ZeppelinhubMessage.class);
} catch (JsonSyntaxException ex) {
LOG.error("Cannot deserialize zeppelinhub message", ex);
msg = EMPTY;
}
return msg;
}
}

62
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/scheduler/SchedulerService.java Normal file
View file

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Creates a thread pool that can schedule zeppelinhub commands.
*
*/
public class SchedulerService {
private final ScheduledExecutorService pool;
private static SchedulerService instance = null;
private SchedulerService(int numberOfThread) {
pool = Executors.newScheduledThreadPool(numberOfThread);
}
public static SchedulerService create(int numberOfThread) {
if (instance == null) {
instance = new SchedulerService(numberOfThread);
}
return instance;
}
public static SchedulerService getInstance() {
if (instance == null) {
instance = new SchedulerService(2);
}
return instance;
}
public void add(Runnable service, int firstExecution, int period) {
pool.scheduleAtFixedRate(service, firstExecution, period, TimeUnit.SECONDS);
}
public void addOnce(Runnable service, int firstExecution) {
pool.schedule(service, firstExecution, TimeUnit.SECONDS);
}
public void close() {
pool.shutdown();
}
}

44
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/scheduler/ZeppelinHeartbeat.java Normal file
View file

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Routine that sends PING to all connected Zeppelin ws connections.
*
*/
public class ZeppelinHeartbeat implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubHeartbeat.class);
private ZeppelinClient client;
public static ZeppelinHeartbeat newInstance(ZeppelinClient client) {
return new ZeppelinHeartbeat(client);
}
private ZeppelinHeartbeat(ZeppelinClient client) {
this.client = client;
}
@Override
public void run() {
LOG.debug("Sending PING to all connected Zeppelin notes");
client.pingAllNotes();
}
}

45
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/scheduler/ZeppelinHubHeartbeat.java Normal file
View file

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.scheduler;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils.ZeppelinhubUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Routine that send PING event to zeppelinhub.
*
*/
public class ZeppelinHubHeartbeat implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubHeartbeat.class);
private ZeppelinhubClient client;
public static ZeppelinHubHeartbeat newInstance(ZeppelinhubClient client) {
return new ZeppelinHubHeartbeat(client);
}
private ZeppelinHubHeartbeat(ZeppelinhubClient client) {
this.client = client;
}
@Override
public void run() {
LOG.debug("Sending PING to zeppelinhub");
client.send(ZeppelinhubUtils.pingMessage(client.getToken()));
}
}

63
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/session/ZeppelinhubSession.java Normal file
View file

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.session;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.websocket.api.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Zeppelinhub session.
*/
public class ZeppelinhubSession {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinhubSession.class);
private Session session;
private final String token;
public static final ZeppelinhubSession EMPTY = new ZeppelinhubSession(null, StringUtils.EMPTY);
public static ZeppelinhubSession createInstance(Session session, String token) {
return new ZeppelinhubSession(session, token);
}
private ZeppelinhubSession(Session session, String token) {
this.session = session;
this.token = token;
}
public boolean isSessionOpen() {
return ((session != null) && (session.isOpen()));
}
public void close() {
if (isSessionOpen()) {
session.close();
}
}
public void sendByFuture(String msg) {
if (StringUtils.isBlank(msg)) {
LOG.error("Cannot send event to Zeppelinhub, msg is empty");
}
if (isSessionOpen()) {
session.getRemote().sendStringByFuture(msg);
} else {
LOG.error("Cannot send event to Zeppelinhub, session is close");
}
}
}

98
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/utils/ZeppelinhubUtils.java Normal file
View file

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.utils;
import java.util.HashMap;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinHubOp;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol.ZeppelinhubMessage;
import org.apache.zeppelin.notebook.socket.Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Helper class.
*
*/
public class ZeppelinhubUtils {
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinhubUtils.class);
public static String liveMessage(String token) {
if (StringUtils.isBlank(token)) {
LOG.error("Cannot create Live message: token is null or empty");
return ZeppelinhubMessage.EMPTY.serialize();
}
HashMap<String, Object> data = new HashMap<String, Object>();
data.put("token", token);
return ZeppelinhubMessage
.newMessage(ZeppelinHubOp.LIVE, data, new HashMap<String, String>())
.serialize();
}
public static String deadMessage(String token) {
if (StringUtils.isBlank(token)) {
LOG.error("Cannot create Dead message: token is null or empty");
return ZeppelinhubMessage.EMPTY.serialize();
}
HashMap<String, Object> data = new HashMap<String, Object>();
data.put("token", token);
return ZeppelinhubMessage
.newMessage(ZeppelinHubOp.DEAD, data, new HashMap<String, String>())
.serialize();
}
public static String pingMessage(String token) {
if (StringUtils.isBlank(token)) {
LOG.error("Cannot create Ping message: token is null or empty");
return ZeppelinhubMessage.EMPTY.serialize();
}
HashMap<String, Object> data = new HashMap<String, Object>();
data.put("token", token);
return ZeppelinhubMessage
.newMessage(ZeppelinHubOp.PING, data, new HashMap<String, String>())
.serialize();
}
public static ZeppelinHubOp toZeppelinHubOp(String text) {
ZeppelinHubOp hubOp = null;
try {
hubOp = ZeppelinHubOp.valueOf(text);
} catch (IllegalArgumentException e) {
// in case of non Hub op
}
return hubOp;
}
public static boolean isZeppelinHubOp(String text) {
return (toZeppelinHubOp(text) != null);
}
public static Message.OP toZeppelinOp(String text) {
Message.OP zeppelinOp = null;
try {
zeppelinOp = Message.OP.valueOf(text);
} catch (IllegalArgumentException e) {
// in case of non Hub op
}
return zeppelinOp;
}
public static boolean isZeppelinOp(String text) {
return (toZeppelinOp(text) != null);
}
}

2
zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.socket;
package org.apache.zeppelin.notebook.socket;
import java.util.HashMap;
import java.util.Map;

22
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -41,12 +41,34 @@ public class ParagraphTest {
text = "%table 1234567";
assertEquals("1234567", Paragraph.getScriptBody(text));
}
@Test
public void scriptBodyWithoutReplName() {
String text = "12345678";
assertEquals(text, Paragraph.getScriptBody(text));
}
@Test
public void replNameEndsWithWhitespace() {
String text = "%md\r\n###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\t###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\u000b###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\f###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md\n###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
text = "%md ###Hello";
assertEquals("md", Paragraph.getRequiredReplName(text));
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given

150
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java Normal file
View file

@ -0,0 +1,150 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.httpclient.HttpException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler;
import org.junit.Before;
import org.junit.Test;
import com.google.common.io.Files;
public class ZeppelinHubRepoTest {
final String TOKEN = "AAA-BBB-CCC-00";
final String testAddr = "http://zeppelinhub.ltd";
private ZeppelinHubRepo repo;
private File pathOfNotebooks = new File(System.getProperty("user.dir") + "/src/test/resources/list_of_notes");
private File pathOfNotebook = new File(System.getProperty("user.dir") + "/src/test/resources/note");
@Before
public void setUp() throws Exception {
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr);
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_TOKEN, "AAA-BBB-CCC-00");
ZeppelinConfiguration conf = new ZeppelinConfiguration();
repo = new ZeppelinHubRepo(conf);
repo.setZeppelinhubRestApiHandler(getMockedZeppelinHandler());
}
private ZeppelinhubRestApiHandler getMockedZeppelinHandler() throws HttpException, IOException {
ZeppelinhubRestApiHandler mockedZeppelinhubHandler = mock(ZeppelinhubRestApiHandler.class);
byte[] response = Files.toByteArray(pathOfNotebooks);
when(mockedZeppelinhubHandler.asyncGet("")).thenReturn(new String(response));
response = Files.toByteArray(pathOfNotebook);
when(mockedZeppelinhubHandler.asyncGet("AAAAA")).thenReturn(new String(response));
return mockedZeppelinhubHandler;
}
@Test
public void testGetZeppelinhubUrl() {
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr);
ZeppelinConfiguration config = new ZeppelinConfiguration();
ZeppelinHubRepo repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "yolow");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("https://www.zeppelinhub.com");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:4242");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd:4242");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:0");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinHubUrl(config)).isEqualTo("http://zeppelinhub.ltd");
}
@Test
public void testGetZeppelinHubWsEndpoint() {
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, testAddr);
ZeppelinConfiguration config = new ZeppelinConfiguration();
ZeppelinHubRepo repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://zeppelinhub.ltd:80/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://zeppelinhub.ltd");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://zeppelinhub.ltd:443/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "yolow");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:443/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://zeppelinhub.ltd:4242");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://zeppelinhub.ltd:4242/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://www.zeppelinhub.com");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:443/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "http://www.zeppelinhub.com");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("ws://www.zeppelinhub.com:80/async");
System.setProperty(ZeppelinHubRepo.ZEPPELIN_CONF_PROP_NAME_SERVER, "https://www.zeppelinhub.com:4242");
config = new ZeppelinConfiguration();
repository = new ZeppelinHubRepo(config);
assertThat(repository.getZeppelinhubWebsocketUri(config)).isEqualTo("wss://www.zeppelinhub.com:4242/async");
}
@Test
public void testGetAllNotes() throws IOException {
List<NoteInfo> notebooks = repo.list();
assertThat(notebooks).isNotEmpty();
assertThat(notebooks.size()).isEqualTo(3);
}
@Test
public void testGetNote() throws IOException {
Note notebook = repo.get("AAAAA");
assertThat(notebook).isNotNull();
assertThat(notebook.id()).isEqualTo("2A94M5J1Z");
}
@Test
public void testRemoveNote() throws IOException {
// not suppose to throw
repo.remove("AAAAA");
}
@Test
public void testRemoveNoteError() throws IOException {
// not suppose to throw
repo.remove("BBBBB");
}
}

123
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinClientTest.java Normal file
View file

@ -0,0 +1,123 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
import static org.junit.Assert.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock.MockEchoWebsocketServer;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.eclipse.jetty.websocket.api.Session;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
public class ZeppelinClientTest {
private Logger LOG = LoggerFactory.getLogger(ZeppelinClientTest.class);
private final int zeppelinPort = 8080;
private final String validWebsocketUrl = "ws://localhost:" + zeppelinPort + "/ws";
private ExecutorService executor;
private MockEchoWebsocketServer echoServer;
@Before
public void setUp() throws Exception {
startWebsocketServer();
}
@After
public void tearDown() throws Exception {
//tear down routine
echoServer.stop();
executor.shutdown();
}
private void startWebsocketServer() throws InterruptedException {
// mock zeppelin websocket server setup
executor = Executors.newFixedThreadPool(1);
echoServer = new MockEchoWebsocketServer(zeppelinPort);
executor.submit(echoServer);
}
@Test
public void zeppelinConnectionTest() {
try {
// Wait for websocket server to start
Thread.sleep(2000);
} catch (InterruptedException e) {
LOG.warn("Cannot wait for websocket server to start, returning");
return;
}
// Initialize and start Zeppelin client
ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "dummy token", null);
client.start();
LOG.info("Zeppelin websocket client started");
// Connection to note AAAA
Session connectionA = client.getZeppelinConnection("AAAA");
assertNotNull(connectionA);
assertTrue(connectionA.isOpen());
assertEquals(client.countConnectedNotes(), 1);
assertEquals(connectionA, client.getZeppelinConnection("AAAA"));
// Connection to note BBBB
Session connectionB = client.getZeppelinConnection("BBBB");
assertNotNull(connectionB);
assertTrue(connectionB.isOpen());
assertEquals(client.countConnectedNotes(), 2);
assertEquals(connectionB, client.getZeppelinConnection("BBBB"));
// Remove connection to note AAAA
client.removeZeppelinConnection("AAAA");
assertEquals(client.countConnectedNotes(), 1);
assertNotEquals(connectionA, client.getZeppelinConnection("AAAA"));
assertEquals(client.countConnectedNotes(), 2);
client.stop();
}
@Test
public void zeppelinClientSingletonTest() {
ZeppelinClient client1 = ZeppelinClient.getInstance();
if (client1 == null) {
client1 = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null);
}
assertNotNull(client1);
ZeppelinClient client2 = ZeppelinClient.getInstance();
assertNotNull(client2);
assertEquals(client1, client2);
}
@Test
public void zeppelinMessageSerializationTest() {
Message msg = new Message(OP.LIST_NOTES);
msg.data = Maps.newHashMap();
msg.data.put("key", "value");
ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null);
String serializedMsg = client.serialize(msg);
Message deserializedMsg = client.deserialize(serializedMsg);
assertEquals(msg.op, deserializedMsg.op);
assertEquals(msg.data.get("key"), deserializedMsg.data.get("key"));
String invalidMsg = "random text";
deserializedMsg =client.deserialize(invalidMsg);
assertNull(deserializedMsg);
}
@Test
public void sendToZeppelinTest() {
ZeppelinClient client = ZeppelinClient.initialize(validWebsocketUrl, "TOKEN", null);
client.start();
Message msg = new Message(OP.LIST_NOTES);
msg.data = Maps.newHashMap();
msg.data.put("key", "value");
client.send(msg, "DDDD");
client.removeZeppelinConnection("DDDD");
client.stop();
}
}

72
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/ZeppelinhubClientTest.java Normal file
View file

@ -0,0 +1,72 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket;
import static org.junit.Assert.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.ZeppelinhubClient;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock.MockEchoWebsocketServer;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ZeppelinhubClientTest {
private Logger LOG = LoggerFactory.getLogger(ZeppelinClientTest.class);
private final int zeppelinPort = 8090;
private final String validWebsocketUrl = "ws://localhost:" + zeppelinPort + "/ws";
private ExecutorService executor;
private MockEchoWebsocketServer echoServer;
private final String runNotebookMsg =
"{\"op\":\"RUN_NOTEBOOK\"," +
"\"data\":[{\"id\":\"20150112-172845_1301897143\",\"title\":null,\"config\":{},\"params\":{},\"data\":null}," +
"{\"id\":\"20150112-172845_1301897143\",\"title\":null,\"config\":{},\"params\":{},\"data\":null}]," +
"\"meta\":{\"owner\":\"author\",\"instance\":\"my-zepp\",\"noteId\":\"2AB7SY361\"}}";
private final String invalidRunNotebookMsg = "some random string";
@Before
public void setUp() throws Exception {
startWebsocketServer();
}
@After
public void tearDown() throws Exception {
//tear down routine
echoServer.stop();
executor.shutdown();
}
private void startWebsocketServer() throws InterruptedException {
// mock zeppelin websocket server setup
executor = Executors.newFixedThreadPool(1);
echoServer = new MockEchoWebsocketServer(zeppelinPort);
executor.submit(echoServer);
}
@Test
public void zeppelinhubClientSingletonTest() {
ZeppelinhubClient client1 = ZeppelinhubClient.getInstance();
if (client1 == null) {
client1 = ZeppelinhubClient.initialize(validWebsocketUrl, "TOKEN");
}
assertNotNull(client1);
ZeppelinhubClient client2 = ZeppelinhubClient.getInstance();
assertNotNull(client2);
assertEquals(client1, client2);
}
@Test
public void runAllParagraphTest() throws Exception {
Client.initialize(validWebsocketUrl, validWebsocketUrl, "TOKEN", null);
Client.getInstance().start();
ZeppelinhubClient zeppelinhubClient = ZeppelinhubClient.getInstance();
boolean runStatus = zeppelinhubClient.runAllParagraph("2AB7SY361", runNotebookMsg);
assertTrue(runStatus);
runStatus = zeppelinhubClient.runAllParagraph("2AB7SY361", invalidRunNotebookMsg);
assertFalse(runStatus);
Client.getInstance().stop();
}
}

46
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEchoWebsocketServer.java Normal file
View file

@ -0,0 +1,46 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.slf4j.LoggerFactory;
public class MockEchoWebsocketServer implements Runnable {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MockEchoWebsocketServer.class);
private Server server;
public MockEchoWebsocketServer(int port) {
server = new Server();
ServerConnector connector = new ServerConnector(server);
connector.setPort(port);
server.addConnector(connector);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
server.setHandler(context);
//ServletHolder holderEvents = new ServletHolder("ws-events", MockEventServlet.class);
context.addServlet(MockEventServlet.class, "/ws/*");
}
public void start() throws Exception {
LOG.info("Starting mock echo websocket server");
server.start();
server.join();
}
public void stop() throws Exception {
LOG.info("Stopping mock echo websocket server");
server.stop();
}
@Override
public void run() {
try {
this.start();
} catch (Exception e) {
LOG.error("Couldn't start mock echo websocket server", e);
}
}
}

14
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventServlet.java Normal file
View file

@ -0,0 +1,14 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock;
import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
@SuppressWarnings("serial")
public class MockEventServlet extends WebSocketServlet
{
@Override
public void configure(WebSocketServletFactory factory)
{
factory.register(MockEventSocket.class);
}
}

38
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/mock/MockEventSocket.java Normal file
View file

@ -0,0 +1,38 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.mock;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MockEventSocket extends WebSocketAdapter {
private static final Logger LOG = LoggerFactory.getLogger(MockEventServlet.class);
private Session session;
@Override
public void onWebSocketConnect(Session session) {
super.onWebSocketConnect(session);
this.session = session;
LOG.info("Socket Connected: " + session);
}
@Override
public void onWebSocketText(String message) {
super.onWebSocketText(message);
session.getRemote().sendStringByFuture(message);
LOG.info("Received TEXT message: {}", message);
}
@Override
public void onWebSocketClose(int statusCode, String reason) {
super.onWebSocketClose(statusCode, reason);
LOG.info("Socket Closed: [{}] {}", statusCode, reason);
}
@Override
public void onWebSocketError(Throwable cause) {
super.onWebSocketError(cause);
LOG.error("Websocket error: {}", cause);
}
}

43
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/websocket/protocol/ZeppelinhubMessageTest.java Normal file
View file

@ -0,0 +1,43 @@
package org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.protocol;
import static org.junit.Assert.assertEquals;
import java.util.Map;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.junit.Test;
import com.google.common.collect.Maps;
public class ZeppelinhubMessageTest {
private String msg = "{\"op\":\"LIST_NOTES\",\"data\":\"my data\",\"meta\":{\"key1\":\"val1\"}}";
@Test
public void testThatCanSerializeZeppelinHubMessage() {
Map<String,String> meta = Maps.newHashMap();
meta.put("key1", "val1");
String zeppelinHubMsg = ZeppelinhubMessage.newMessage(OP.LIST_NOTES, "my data", meta).serialize();
assertEquals(msg, zeppelinHubMsg);
}
@Test
public void testThastCanDeserialiseZeppelinhubMessage() {
Map<String,String> meta = Maps.newHashMap();
meta.put("key1", "val1");
ZeppelinhubMessage expected = ZeppelinhubMessage.newMessage(OP.LIST_NOTES.toString(), "my data", meta);
ZeppelinhubMessage zeppelinHubMsg = ZeppelinhubMessage.deserialize(msg);
assertEquals(expected.op, zeppelinHubMsg.op);
assertEquals(expected.data, zeppelinHubMsg.data);
assertEquals(expected.meta, zeppelinHubMsg.meta);
}
@Test
public void testThatInvalidStringReturnEmptyZeppelinhubMessage() {
assertEquals(ZeppelinhubMessage.EMPTY, ZeppelinhubMessage.deserialize(""));
assertEquals(ZeppelinhubMessage.EMPTY, ZeppelinhubMessage.deserialize("dwfewewrewr"));
}
}

41
zeppelin-zengine/src/test/resources/list_of_notes Normal file
View file

@ -0,0 +1,41 @@
[
{
"id": "2ABSFSR35",
"name": "ES RDD",
"config": {
"looknfeel": "default",
"codeHighlightStyle": "GitHub",
"codeHighlightStyleOrig": "GitHub"
},
"info": {},
"paragraphs": [],
"angularObjects": {},
"lastUpdate": 1445663884000
},
{
"id": "2A94M5J1Z",
"name": "Zeppelin Tutorial",
"config": {
"looknfeel": "default"},
"info": {},
"paragraphs": [],
"angularObjects": {
"2AMFHG1GQ": [],
"2APT3NC5T": [],
"2ANPJTJRQ": [],
"2AMJZ9R4C": [],
"2ANJFBYJS": [],
"2APKU6T1J": []},
"lastUpdate": 1446688883000
},
{
"id": "2AVSEYW1R",
"name": "NFLabs - Tracking beta",
"config": {
"looknfeel": "default"},
"info": {},
"paragraphs": [],
"angularObjects": {},
"lastUpdate": 1445663888000
}
]

16
zeppelin-zengine/src/test/resources/note Normal file
View file

@ -0,0 +1,16 @@
{
"id": "2A94M5J1Z",
"name": "Zeppelin Tutorial",
"config": {
"looknfeel": "default"},
"info": {},
"paragraphs": [],
"angularObjects": {
"2AMFHG1GQ": [],
"2APT3NC5T": [],
"2ANPJTJRQ": [],
"2AMJZ9R4C": [],
"2ANJFBYJS": [],
"2APKU6T1J": []},
"lastUpdate": 1446688883000
}