Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'apache/master' into keycloak

Browse source
This commit is contained in:
andrea 2017-06-19 20:54:30 +01:00
commit 31270f2cdc
159 changed files with 5631 additions and 1292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.travis.yml
View file

@ -46,8 +46,12 @@ matrix:
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test core modules
#
# Several tests were excluded from this configuration due to the following issues:
# HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
# After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false"
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk7"
@ -55,7 +59,7 @@ matrix:
# Test interpreter modules
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
# Test spark module for 2.1.0 with scala 2.11, livy
- jdk: "oraclejdk7"
@ -89,7 +93,7 @@ before_install:
- changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE 2>/dev/null) || changedfiles=""
- echo $changedfiles
- hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true);
- gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog=""
- gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog=""
- clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
- if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache clear; else echo "Using cached bower_components."; fi
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc

25
conf/shiro.ini.template
View file

@ -18,7 +18,8 @@
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password1, admin
# To enable admin user, uncomment the following line and set an appropriate password.
#admin = password1, admin
user1 = password2, role1, role2
user2 = password3, role3
user3 = password4, role2
@ -73,14 +74,22 @@ role3 = *
admin = *
[urls]
# This section is used for url-based security.
# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide.
# This section is used for url-based security. For details see the shiro.ini documentation.
#
# You can secure interpreter, configuration and credential information by urls.
# Comment or uncomment the below urls that you want to hide:
# anon means the access is anonymous.
# authc means Form based Auth Security
# To enfore security, comment the line below and uncomment the next one
# authc means form based auth Security.
#
# IMPORTANT: Order matters: URL path expressions are evaluated against an incoming request
# in the order they are defined and the FIRST MATCH WINS.
#
# To allow anonymous access to all but the stated urls,
# uncomment the line second last line (/** = anon) and comment the last line (/** = authc)
#
/api/version = anon
#/api/interpreter/** = authc, roles[admin]
#/api/configurations/** = authc, roles[admin]
#/api/credential/** = authc, roles[admin]
/api/interpreter/** = authc, roles[admin]
/api/configurations/** = authc, roles[admin]
/api/credential/** = authc, roles[admin]
#/** = anon
/** = authc

4
conf/zeppelin-env.cmd.template
View file

@ -39,7 +39,9 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading
REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader
REM set ZEPPELIN_HELIUM_NODE_INSTALLER_URL REM Remote Node installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_NPM_INSTALLER_URL REM Remote Npm installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL REM Remote Yarn package installer url for Helium dependency loader
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?

4
conf/zeppelin-env.sh.template
View file

@ -44,7 +44,9 @@
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading
# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader
# export ZEPPELIN_HELIUM_NODE_INSTALLER_URL # Remote Node installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_NPM_INSTALLER_URL # Remote Npm installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL # Remote Yarn package installer url for Helium dependency loader
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise

16
conf/zeppelin-site.xml.template
View file

@ -252,9 +252,21 @@
</property>
<property>
<name>zeppelin.helium.npm.registry</name>
<name>zeppelin.helium.node.installer.url</name>
<value>https://nodejs.org/dist/</value>
<description>Remote Node installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.npm.installer.url</name>
<value>http://registry.npmjs.org/</value>
<description>Remote Npm registry for Helium dependency loader</description>
<description>Remote Npm installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.yarnpkg.installer.url</name>
<value>https://github.com/yarnpkg/yarn/releases/download/</value>
<description>Remote Yarn package installer url for Helium dependency loader</description>
</property>
<property>

3
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -88,6 +88,7 @@
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#text">Text</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#html">Html</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table">Table</a></li>
<li><a href="{{BASE_PATH}}/displaysystem/basicdisplaysystem.html#network">Network</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Angular API</b><span></li>
<li><a href="{{BASE_PATH}}/displaysystem/back-end-angular.html">Angular (backend API)</a></li>
@ -105,6 +106,7 @@
<li><a href="{{BASE_PATH}}/storage/storage.html#notebook-storage-in-mongodb">MongoDB Storage</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>REST API</b><span></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-zeppelin-server.html">Zeppelin Server API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-interpreter.html">Interpreter API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-notebook.html">Notebook API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-notebookRepo.html">Notebook Repository API</a></li>
@ -116,6 +118,7 @@
<li><a href="{{BASE_PATH}}/security/shiroauthentication.html">Shiro Authentication</a></li>
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
<li><a href="{{BASE_PATH}}/security/helium_authorization.html">Helium Authorization</a></li>
<li role="separator" class="divider"></li>
<li class="title"><span><b>Helium Framework (Experimental)</b></span></li>
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>

BIN
docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 48 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 52 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 41 KiB

BIN
docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 31 KiB

101
docs/displaysystem/basicdisplaysystem.md
View file

@ -61,3 +61,104 @@ If table contents start with `%html`, it is interpreted as an HTML.
<img src="../assets/themes/zeppelin/img/screenshots/display_table_html.png" />
> **Note :** Display system is backend independent.
## Network
With the `%network` directive, Zeppelin treats your output as a graph. Zeppelin can leverage the Property Graph Model.
### What is the Labelled Property Graph Model?
A [Property Graph](https://github.com/tinkerpop/gremlin/wiki/Defining-a-Property-Graph) is a graph that has these elements:
* a set of vertices
* each vertex has a unique identifier.
* each vertex has a set of outgoing edges.
* each vertex has a set of incoming edges.
* each vertex has a collection of properties defined by a map from key to value
* a set of edges
* each edge has a unique identifier.
* each edge has an outgoing tail vertex.
* each edge has an incoming head vertex.
* each edge has a label that denotes the type of relationship between its two vertices.
* each edge has a collection of properties defined by a map from key to value.
<img src="https://github.com/tinkerpop/gremlin/raw/master/doc/images/graph-example-1.jpg" />
A [Labelled Property Graph](https://neo4j.com/developer/graph-database/#property-graph) is a Property Graph where the nodes can be tagged with **labels** representing their different roles in the graph model
<img src="http://s3.amazonaws.com/dev.assets.neo4j.com/wp-content/uploads/property_graph_model.png" />
### What are the APIs?
The new NETWORK visualization is based on json with the following params:
* "nodes" (mandatory): list of nodes of the graph every node can have the following params:
* "id" (mandatory): the id of the node (must be unique);
* "label": the main Label of the node;
* "labels": the list of the labels of the node;
* "data": the data attached to the node;
* "edges": list of the edges of the graph;
* "id" (mandatory): the id of the edge (must be unique);
* "source" (mandatory): the id of source node of the edge;
* "target" (mandatory): the id of target node of the edge;
* "label": the main type of the edge;
* "data": the data attached to the edge;
* "labels": a map (K, V) where K is the node label and V is the color of the node;
* "directed": (true/false, default false) wich tells if is directed graph or not;
* "types": a *distinct* list of the edge types of the graph
If you click on a node or edge on the bottom of the paragraph you find a list of entity properties
<img src="../assets/themes/zeppelin/img/screenshots/display_network.png" />
This kind of graph can be easily *flatten* in order to support other visualization formats provided by Zeppelin.
<img src="../assets/themes/zeppelin/img/screenshots/display_network_flatten.png" />
### How to use it?
An example of a simple graph
```
%spark
print(s"""
%network {
"nodes": [
{"id": 1},
{"id": 2},
{"id": 3}
],
"edges": [
{"source": 1, "target": 2, "id" : 1},
{"source": 2, "target": 3, "id" : 2},
{"source": 1, "target": 2, "id" : 3},
{"source": 1, "target": 2, "id" : 4},
{"source": 2, "target": 1, "id" : 5},
{"source": 2, "target": 1, "id" : 6}
]
}
""")
```
that will look like:
<img src="../assets/themes/zeppelin/img/screenshots/display_simple_network.png" />
A little more complex graph:
```
%spark
print(s"""
%network {
"nodes": [{"id": 1, "label": "User", "data": {"fullName":"Andrea Santurbano"}},{"id": 2, "label": "User", "data": {"fullName":"Lee Moon Soo"}},{"id": 3, "label": "Project", "data": {"name":"Zeppelin"}}],
"edges": [{"source": 2, "target": 1, "id" : 1, "label": "HELPS"},{"source": 2, "target": 3, "id" : 2, "label": "CREATE"},{"source": 1, "target": 3, "id" : 3, "label": "CONTRIBUTE_TO", "data": {"oldPR": "https://github.com/apache/zeppelin/pull/1582"}}],
"labels": {"User": "#8BC34A", "Project": "#3071A9"},
"directed": true,
"types": ["HELPS", "CREATE", "CONTRIBUTE_TO"]
}
""")
```
that will look like:
<img src="../assets/themes/zeppelin/img/screenshots/display_complex_network.png" />

2
docs/index.md
View file

@ -161,6 +161,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* [Azure Storage](./storage/storage.html#notebook-storage-in-azure)
* [ZeppelinHub Storage](./storage/storage.html#storage-in-zeppelinhub)
* REST API: available REST API list in Apache Zeppelin
* [Zeppelin server API](./rest-api/rest-zeppelin-server.html)
* [Interpreter API](./rest-api/rest-interpreter.html)
* [Notebook API](./rest-api/rest-notebook.html)
* [Notebook Repository API](./rest-api/rest-notebookRepo.html)
@ -172,6 +173,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
* [Shiro Authentication](./security/shiroauthentication.html)
* [Notebook Authorization](./security/notebook_authorization.html)
* [Data Source Authorization](./security/datasource_authorization.html)
* [Helium Authorization](./security/helium_authorization.html)
* Helium Framework (Experimental)
* [Writing Zeppelin Application](./development/writingzeppelinapplication.html)
* [Writing Zeppelin Spell](./development/writingzeppelinspell.html)

18
docs/install/configuration.md
View file

@ -282,10 +282,22 @@ If both are defined, then the **environment variables** will take priority.
<td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td>
<td><h6 class="properties">zeppelin.helium.npm.registry</h6></td>
<td><h6 class="properties">ZEPPELIN_HELIUM_NODE_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.node.installer.url</h6></td>
<td>https://nodejs.org/dist/</td>
<td>Remote Node installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_NPM_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.npm.installer.url</h6></td>
<td>http://registry.npmjs.org/</td>
<td>Remote Npm registry for Helium dependency loader</td>
<td>Remote Npm installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.yarnpkg.installer.url</h6></td>
<td>https://github.com/yarnpkg/yarn/releases/download/</td>
<td>Remote Yarn package installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td>

10
docs/interpreter/jdbc.md
View file

@ -128,6 +128,16 @@ The JDBC interpreter properties are defined by default like below.
<td></td>
<td>Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)</td>
</tr>
<tr>
<td>default.completer.ttlInSeconds</td>
<td>120</td>
<td>Time to live sql completer in seconds (-1 to update everytime, 0 to disable update)</td>
</tr>
<tr>
<td>default.splitQueries</td>
<td>false</td>
<td>Each query is executed apart and returns the result</td>
</tr>
</table>
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.

1
docs/rest-api/rest-configuration.md
View file

@ -29,7 +29,6 @@ All REST APIs are available starting with the following endpoint `http://[zeppel
Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html).
nd a need for an additional REST API, please [file an issue or send us mail](../../community.html).
## Configuration REST API list

78
docs/rest-api/rest-zeppelin-server.md Normal file
View file

@ -0,0 +1,78 @@
---
layout: page
title: "Apache Zeppelin Server REST API"
description: "This page contains Apache Zeppelin Server REST API information."
group: rest-api
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Apache Zeppelin Server REST API
<div id="toc"></div>
## Overview
Apache Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html).
## Zeppelin Server REST API list
### Change the log level of Zeppelin Server
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```PUT``` method is used to update the root logger's log level of the server.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/log/level/<LOG_LEVEL>```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 406 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>
<pre>
{
"status": "OK"
}
</pre>
</td>
</tr>
<tr>
<td> sample error JSON response
</td>
<td>
<pre>
{
"status":"NOT_ACCEPTABLE",
"message":"Please check LOG level specified. Valid values: DEBUG, ERROR, FATAL, INFO, TRACE, WARN"
}
</pre>
</td>
</tr>
</table>

28
docs/security/helium_authorization.md Normal file
View file

@ -0,0 +1,28 @@
---
layout: page
title: "Helium Authorization in Apache Zeppelin"
description: "Apache Zeppelin supports Helium plugins which fetch required installer packages from remote registry/repositories"
group: security
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Helium Authorization in Apache Zeppelin
<div id="toc"></div>
## How to configure proxies?
Set **http_proxy** and **https_proxy** env variables to allow connection to npm registry behind a corporate firewall.

44
docs/security/shiroauthentication.md
View file

@ -128,6 +128,10 @@ Change the following values in the Shiro.ini file, and uncomment the line:
### LDAP
Two options exist for configuring an LDAP Realm. The simpler to use is the LdapGroupRealm. How ever it has limited
flexibility with mapping of ldap groups to users and for authorization for user groups. A sample configuration file for
this realm is given below.
```
ldapRealm = org.apache.zeppelin.realm.LdapGroupRealm
# search base for ldap groups (only relevant for LdapGroupRealm):
@ -137,6 +141,46 @@ ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
ldapRealm.contextFactory.authenticationMechanism = simple
```
The other more flexible option is to use the LdapRealm. It allows for mapping of ldapgroups to roles and also allows for
role/group based authentication into the zeppelin server. Sample configuration for this realm is given below.
```
[main]
ldapRealm=org.apache.zeppelin.realm.LdapRealm
ldapRealm.contextFactory.authenticationMechanism=simple
ldapRealm.contextFactory.url=ldap://localhost:33389
ldapRealm.userDnTemplate=uid={0},ou=people,dc=hadoop,dc=apache,dc=org
# Ability to set ldap paging Size if needed default is 100
ldapRealm.pagingSize = 200
ldapRealm.authorizationEnabled=true
ldapRealm.contextFactory.systemAuthenticationMechanism=simple
ldapRealm.searchBase=dc=hadoop,dc=apache,dc=org
ldapRealm.userSearchBase = dc=hadoop,dc=apache,dc=org
ldapRealm.groupSearchBase = ou=groups,dc=hadoop,dc=apache,dc=org
ldapRealm.groupObjectClass=groupofnames
# Allow userSearchAttribute to be customized
ldapRealm.userSearchAttributeName = sAMAccountName
ldapRealm.memberAttribute=member
# force usernames returned from ldap to lowercase useful for AD
ldapRealm.userLowerCase = true
# ability set searchScopes subtree (default), one, base
ldapRealm.userSearchScope = subtree;
ldapRealm.groupSearchScope = subtree;
ldapRealm.memberAttributeValueTemplate=cn={0},ou=people,dc=hadoop,dc=apache,dc=org
ldapRealm.contextFactory.systemUsername=uid=guest,ou=people,dc=hadoop,dc=apache,dc=org
ldapRealm.contextFactory.systemPassword=S{ALIAS=ldcSystemPassword}
# enable support for nested groups using the LDAP_MATCHING_RULE_IN_CHAIN operator
ldapRealm.groupSearchEnableMatchingRuleInChain = true
# optional mapping from physical groups to logical application roles
ldapRealm.rolesByGroup = LDN_USERS: user_role, NYK_USERS: user_role, HKG_USERS: user_role, GLOBAL_ADMIN: admin_role
# optional list of roles that are allowed to authenticate. Incase not present all groups are allowed to authenticate (login).
# This changes nothing for url specific permissions that will continue to work as specified in [urls].
ldapRealm.allowedRolesForAuthentication = admin_role,user_role
ldapRealm.permissionsByRole= user_role = *:ToDoItemsJdo:*:*, *:ToDoItem:*:*; admin_role = *
securityManager.sessionManager = $sessionManager
securityManager.realms = $ldapRealm
```
### PAM
[PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication
moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can

96
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -23,11 +23,15 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.dbcp2.ConnectionFactory;
import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
@ -101,6 +105,9 @@ public class JDBCInterpreter extends Interpreter {
static final String PASSWORD_KEY = "password";
static final String PRECODE_KEY = "precode";
static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters";
static final String COMPLETER_TTL_KEY = "completer.ttlInSeconds";
static final String DEFAULT_COMPLETER_TTL = "120";
static final String SPLIT_QURIES_KEY = "splitQueries";
static final String JDBC_JCEKS_FILE = "jceks.file";
static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
static final String PRECODE_KEY_TEMPLATE = "%s.precode";
@ -128,6 +135,7 @@ public class JDBCInterpreter extends Interpreter {
private final HashMap<String, Properties> basePropretiesMap;
private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
private final HashMap<String, SqlCompleter> sqlCompletersMap;
private int maxLineResults;
@ -135,6 +143,7 @@ public class JDBCInterpreter extends Interpreter {
super(property);
jdbcUserConfigurationsMap = new HashMap<>();
basePropretiesMap = new HashMap<>();
sqlCompletersMap = new HashMap<>();
maxLineResults = MAX_LINE_DEFAULT;
}
@ -188,11 +197,43 @@ public class JDBCInterpreter extends Interpreter {
}
}
private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) {
private SqlCompleter createOrUpdateSqlCompleter(SqlCompleter sqlCompleter,
final Connection connection, String propertyKey, final String buf, final int cursor) {
String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
String filters = getProperty(schemaFiltersKey);
SqlCompleter completer = new SqlCompleter();
completer.initFromConnection(jdbcConnection, filters);
String sqlCompleterTtlKey = String.format("%s.%s", propertyKey, COMPLETER_TTL_KEY);
final String schemaFiltersString = getProperty(schemaFiltersKey);
int ttlInSeconds = Integer.valueOf(
StringUtils.defaultIfEmpty(getProperty(sqlCompleterTtlKey), DEFAULT_COMPLETER_TTL)
);
final SqlCompleter completer;
if (sqlCompleter == null) {
completer = new SqlCompleter(ttlInSeconds);
} else {
completer = sqlCompleter;
}
ExecutorService executorService = Executors.newFixedThreadPool(1);
executorService.execute(new Runnable() {
@Override
public void run() {
completer.createOrUpdateFromConnection(connection, schemaFiltersString, buf, cursor);
}
});
executorService.shutdown();
try {
// protection to release connection
executorService.awaitTermination(3, TimeUnit.SECONDS);
} catch (InterruptedException e) {
logger.warn("Completion timeout", e);
if (connection != null) {
try {
connection.close();
} catch (SQLException e1) {
logger.warn("Error close connection", e1);
}
}
}
return completer;
}
@ -499,7 +540,6 @@ public class JDBCInterpreter extends Interpreter {
StringBuilder query = new StringBuilder();
char character;
Boolean antiSlash = false;
Boolean multiLineComment = false;
Boolean singleLineComment = false;
Boolean quoteString = false;
@ -522,14 +562,8 @@ public class JDBCInterpreter extends Interpreter {
continue;
}
if (character == '\\') {
antiSlash = true;
}
if (character == '\'') {
if (antiSlash) {
antiSlash = false;
} else if (quoteString) {
if (quoteString) {
quoteString = false;
} else if (!doubleQuoteString) {
quoteString = true;
@ -537,9 +571,7 @@ public class JDBCInterpreter extends Interpreter {
}
if (character == '"') {
if (antiSlash) {
antiSlash = false;
} else if (doubleQuoteString) {
if (doubleQuoteString && item > 0) {
doubleQuoteString = false;
} else if (!quoteString) {
doubleQuoteString = true;
@ -559,7 +591,7 @@ public class JDBCInterpreter extends Interpreter {
}
}
if (character == ';' && !antiSlash && !quoteString && !doubleQuoteString) {
if (character == ';' && !quoteString && !doubleQuoteString) {
queries.add(StringUtils.trim(query.toString()));
query = new StringBuilder();
} else if (item == sql.length() - 1) {
@ -596,17 +628,29 @@ public class JDBCInterpreter extends Interpreter {
String paragraphId = interpreterContext.getParagraphId();
String user = interpreterContext.getAuthenticationInfo().getUser();
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
boolean splitQuery = false;
String splitQueryProperty = getProperty(String.format("%s.%s", propertyKey, SPLIT_QURIES_KEY));
if (StringUtils.isNotBlank(splitQueryProperty) && splitQueryProperty.equalsIgnoreCase("true")) {
splitQuery = true;
}
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
try {
connection = getConnection(propertyKey, interpreterContext);
if (connection == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
ArrayList<String> multipleSqlArray = splitSqlQueries(sql);
for (int i = 0; i < multipleSqlArray.size(); i++) {
String sqlToExecute = multipleSqlArray.get(i);
List<String> sqlArray;
if (splitQuery) {
sqlArray = splitSqlQueries(sql);
} else {
sqlArray = Arrays.asList(sql);
}
for (int i = 0; i < sqlArray.size(); i++) {
String sqlToExecute = sqlArray.get(i);
statement = connection.createStatement();
if (statement == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
@ -787,6 +831,10 @@ public class JDBCInterpreter extends Interpreter {
InterpreterContext interpreterContext) {
List<InterpreterCompletion> candidates = new ArrayList<>();
String propertyKey = getPropertyKey(buf);
String sqlCompleterKey =
String.format("%s.%s", interpreterContext.getAuthenticationInfo().getUser(), propertyKey);
SqlCompleter sqlCompleter = sqlCompletersMap.get(sqlCompleterKey);
Connection connection = null;
try {
if (interpreterContext != null) {
@ -796,11 +844,9 @@ public class JDBCInterpreter extends Interpreter {
logger.warn("SQLCompleter will created without use connection");
}
SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey);
if (sqlCompleter != null) {
sqlCompleter.complete(buf, cursor - 1, candidates);
}
sqlCompleter = createOrUpdateSqlCompleter(sqlCompleter, connection, propertyKey, buf, cursor);
sqlCompletersMap.put(sqlCompleterKey, sqlCompleter);
sqlCompleter.complete(buf, cursor, candidates);
return candidates;
}

459
jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
View file

@ -24,6 +24,7 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.zeppelin.completer.CachedCompleter;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.completer.StringsCompleter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@ -33,8 +34,6 @@ import org.slf4j.LoggerFactory;
import jline.console.completer.ArgumentCompleter.ArgumentList;
import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
import static org.apache.commons.lang.StringUtils.isBlank;
/**
* SQL auto complete functionality for the JdbcInterpreter.
*/
@ -42,6 +41,7 @@ public class SqlCompleter {
private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
/**
* Delimiter that can split SQL statement in keyword list
*/
@ -59,23 +59,30 @@ public class SqlCompleter {
/**
* Schema completer
*/
private StringsCompleter schemasCompleter = new StringsCompleter();
private CachedCompleter schemasCompleter;
/**
* Contain different completer with table list for every schema name
*/
private Map<String, StringsCompleter> tablesCompleters = new HashMap<>();
private Map<String, CachedCompleter> tablesCompleters = new HashMap<>();
/**
* Contains different completer with column list for every table name
* Table names store as schema_name.table_name
*/
private Map<String, StringsCompleter> columnsCompleters = new HashMap<>();
private Map<String, CachedCompleter> columnsCompleters = new HashMap<>();
/**
* Completer for sql keywords
*/
private StringsCompleter keywordCompleter = new StringsCompleter();
private CachedCompleter keywordCompleter;
private int ttlInSeconds;
public SqlCompleter(int ttlInSeconds) {
this.ttlInSeconds = ttlInSeconds;
}
public int complete(String buffer, int cursor, List<InterpreterCompletion> candidates) {
@ -95,25 +102,9 @@ public class SqlCompleter {
argumentPosition = argumentList.getArgumentPosition();
}
boolean isColumnAllowed = true;
if (buffer.length() > 0) {
String beforeCursorBuffer = buffer.substring(0,
Math.min(cursor, buffer.length())).toUpperCase();
// check what sql is and where cursor is to allow column completion or not
if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
&& !beforeCursorBuffer.contains(" WHERE "))
isColumnAllowed = false;
}
int complete = completeName(cursorArgument, argumentPosition, candidates,
findAliasesInSQL(argumentList.getArguments()), isColumnAllowed);
findAliasesInSQL(argumentList.getArguments()));
if (candidates.size() == 1) {
InterpreterCompletion interpreterCompletion = candidates.get(0);
interpreterCompletion.setName(interpreterCompletion.getName() + " ");
interpreterCompletion.setValue(interpreterCompletion.getValue() + " ");
candidates.set(0, interpreterCompletion);
}
logger.debug("complete:" + complete + ", size:" + candidates.size());
return complete;
}
@ -132,6 +123,7 @@ public class SqlCompleter {
Set<String> res = new HashSet<>();
try {
ResultSet schemas = meta.getSchemas();
try {
while (schemas.next()) {
String schemaName = schemas.getString("TABLE_SCHEM");
@ -185,58 +177,40 @@ public class SqlCompleter {
return res;
}
private static void fillTableNames(String schema, DatabaseMetaData meta, Set<String> tables) {
try (ResultSet tbls = meta.getTables(schema, schema, "%", null)) {
while (tbls.next()) {
String table = tbls.getString("TABLE_NAME");
tables.add(table);
}
} catch (Throwable t) {
logger.error("Failed to retrieve the table name", t);
}
}
/**
* Fill two map with list of tables and list of columns
*
* @param catalogName name of a catalog
* @param meta metadata from connection to database
* @param schemaFilter a schema name pattern; must match the schema name
* as it is stored in the database; "" retrieves those without a schema;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%'; for example "prod_v_%"
* @param tables function fills this map, for every schema name adds
* set of table names within the schema
* @param columns function fills this map, for every table name adds set
* @param schema name of a scheme
* @param table name of a table
* @param meta meta metadata from connection to database
* @param columns function fills this set, for every table name adds set
* of columns within the table; table name is in format schema_name.table_name
*/
private static void fillTableAndColumnNames(String catalogName, DatabaseMetaData meta,
String schemaFilter,
Map<String, Set<String>> tables,
Map<String, Set<String>> columns) {
try {
ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%");
try {
while (cols.next()) {
String schema = cols.getString("TABLE_SCHEM");
if (schema == null) {
schema = cols.getString("TABLE_CAT");
}
if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) {
continue;
}
String table = cols.getString("TABLE_NAME");
String column = cols.getString("COLUMN_NAME");
if (!isBlank(table)) {
String schemaTable = schema + "." + table;
if (!columns.containsKey(schemaTable)) {
columns.put(schemaTable, new HashSet<String>());
}
columns.get(schemaTable).add(column);
if (!tables.containsKey(schema)) {
tables.put(schema, new HashSet<String>());
}
tables.get(schema).add(table);
}
}
} finally {
cols.close();
private static void fillColumnNames(String schema, String table, DatabaseMetaData meta,
Set<String> columns) {
try (ResultSet cols = meta.getColumns(schema, schema, table, "%")) {
while (cols.next()) {
String column = cols.getString("COLUMN_NAME");
columns.add(column);
}
} catch (Throwable t) {
logger.error("Failed to retrieve the column name", t);
}
}
public static Set<String> getSqlKeywordsCompletions(Connection connection) throws IOException,
public static Set<String> getSqlKeywordsCompletions(DatabaseMetaData meta) throws IOException,
SQLException {
// Add the default SQL completions
@ -246,12 +220,11 @@ public class SqlCompleter {
Set<String> completions = new TreeSet<>();
if (null != connection) {
DatabaseMetaData metaData = connection.getMetaData();
if (null != meta) {
// Add the driver specific SQL completions
String driverSpecificKeywords =
"/" + metaData.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords";
"/" + meta.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords";
logger.info("JDBC DriverName:" + driverSpecificKeywords);
try {
if (SqlCompleter.class.getResource(driverSpecificKeywords) != null) {
@ -269,27 +242,27 @@ public class SqlCompleter {
// Add the keywords from the current JDBC connection
try {
keywords += "," + metaData.getSQLKeywords();
keywords += "," + meta.getSQLKeywords();
} catch (Exception e) {
logger.debug("fail to get SQL key words from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getStringFunctions();
keywords += "," + meta.getStringFunctions();
} catch (Exception e) {
logger.debug("fail to get string function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getNumericFunctions();
keywords += "," + meta.getNumericFunctions();
} catch (Exception e) {
logger.debug("fail to get numeric function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getSystemFunctions();
keywords += "," + meta.getSystemFunctions();
} catch (Exception e) {
logger.debug("fail to get system function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getTimeDateFunctions();
keywords += "," + meta.getTimeDateFunctions();
} catch (Exception e) {
logger.debug("fail to get time date function names from database metadata: " + e, e);
}
@ -307,95 +280,101 @@ public class SqlCompleter {
return completions;
}
/**
* Initializes local schema completers from list of schema names
*
* @param schemas set of schema names
*/
private void initSchemas(Set<String> schemas) {
schemasCompleter = new StringsCompleter(new TreeSet<>(schemas));
}
/**
* Initializes local table completers from list of table name
*
* @param tables for every schema name there is a set of table names within the schema
*/
private void initTables(Map<String, Set<String>> tables) {
tablesCompleters.clear();
for (Map.Entry<String, Set<String>> entry : tables.entrySet()) {
tablesCompleters.put(entry.getKey(), new StringsCompleter(new TreeSet<>(entry.getValue())));
}
}
/**
* Initializes local column completers from list of column names
*
* @param columns for every table name there is a set of columns within the table;
* table name is in format schema_name.table_name
*/
private void initColumns(Map<String, Set<String>> columns) {
columnsCompleters.clear();
for (Map.Entry<String, Set<String>> entry : columns.entrySet()) {
columnsCompleters.put(entry.getKey(), new StringsCompleter(new TreeSet<>(entry.getValue())));
}
}
/**
* Initializes all local completers
*
* @param schemas set of schema names
* @param tables for every schema name there is a set of table names within the schema
* @param columns for every table name there is a set of columns within the table;
* table name is in format schema_name.table_name
* @param keywords set with sql keywords
*/
public void init(Set<String> schemas, Map<String, Set<String>> tables,
Map<String, Set<String>> columns, Set<String> keywords) {
initSchemas(schemas);
initTables(tables);
initColumns(columns);
keywordCompleter = new StringsCompleter(keywords);
}
/**
* Initializes all local completers from database connection
*
* @param connection database connection
* @param schemaFiltersString a comma separated schema name patterns; supports '%' symbol;
* for example "prod_v_%,prod_t_%"
* @param schemaFiltersString a comma separated schema name patterns, supports '%' symbol;
* for example "prod_v_%,prod_t_%"
*/
public void initFromConnection(Connection connection, String schemaFiltersString) {
if (schemaFiltersString == null) {
schemaFiltersString = StringUtils.EMPTY;
}
List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
public void createOrUpdateFromConnection(Connection connection, String schemaFiltersString,
String buffer, int cursor) {
try (Connection c = connection) {
Map<String, Set<String>> tables = new HashMap<>();
Map<String, Set<String>> columns = new HashMap<>();
if (schemaFiltersString == null) {
schemaFiltersString = StringUtils.EMPTY;
}
List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
CursorArgument cursorArgument = parseCursorArgument(buffer, cursor);
Set<String> tables = new HashSet<>();
Set<String> columns = new HashSet<>();
Set<String> schemas = new HashSet<>();
Set<String> catalogs = new HashSet<>();
Set<String> keywords = getSqlKeywordsCompletions(connection);
if (connection != null) {
schemas = getSchemaNames(connection.getMetaData(), schemaFilters);
catalogs = getCatalogNames(connection.getMetaData(), schemaFilters);
if (schemas.size() == 0) {
schemas.addAll(catalogs);
Set<String> keywords = new HashSet<>();
if (c != null) {
DatabaseMetaData databaseMetaData = c.getMetaData();
if (keywordCompleter == null || keywordCompleter.getCompleter() == null
|| keywordCompleter.isExpired()) {
keywords = getSqlKeywordsCompletions(databaseMetaData);
initKeywords(keywords);
}
for (String schema : schemas) {
for (String schemaFilter : schemaFilters) {
fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables,
columns);
if (cursorArgument.needLoadSchemas() &&
(schemasCompleter == null || schemasCompleter.getCompleter() == null
|| schemasCompleter.isExpired())) {
schemas = getSchemaNames(databaseMetaData, schemaFilters);
catalogs = getCatalogNames(databaseMetaData, schemaFilters);
if (schemas.size() == 0) {
schemas.addAll(catalogs);
}
initSchemas(schemas);
}
CachedCompleter tablesCompleter = tablesCompleters.get(cursorArgument.getSchema());
if (cursorArgument.needLoadTables() &&
(tablesCompleter == null || tablesCompleter.isExpired())) {
fillTableNames(cursorArgument.getSchema(), databaseMetaData, tables);
initTables(cursorArgument.getSchema(), tables);
}
String schemaTable =
String.format("%s.%s", cursorArgument.getSchema(), cursorArgument.getTable());
CachedCompleter columnsCompleter = columnsCompleters.get(schemaTable);
if (cursorArgument.needLoadColumns() &&
(columnsCompleter == null || columnsCompleter.isExpired())) {
fillColumnNames(cursorArgument.getSchema(), cursorArgument.getTable(), databaseMetaData,
columns);
initColumns(schemaTable, columns);
}
logger.info("Completer initialized with " + schemas.size() + " schemas, " +
columns.size() + " tables and " + keywords.size() + " keywords");
}
init(schemas, tables, columns, keywords);
logger.info("Completer initialized with " + schemas.size() + " schemas, " +
columns.size() + " tables and " + keywords.size() + " keywords");
} catch (SQLException | IOException e) {
logger.error("Failed to update the metadata conmpletions", e);
logger.error("Failed to update the metadata completions", e);
}
}
public void initKeywords(Set<String> keywords) {
if (keywords != null && !keywords.isEmpty()) {
keywordCompleter = new CachedCompleter(new StringsCompleter(keywords), 0);
}
}
public void initSchemas(Set<String> schemas) {
if (schemas != null && !schemas.isEmpty()) {
schemasCompleter = new CachedCompleter(
new StringsCompleter(new TreeSet<>(schemas)), ttlInSeconds);
}
}
public void initTables(String schema, Set<String> tables) {
if (tables != null && !tables.isEmpty()) {
tablesCompleters.put(schema, new CachedCompleter(
new StringsCompleter(new TreeSet<>(tables)), ttlInSeconds));
}
}
public void initColumns(String schemaTable, Set<String> columns) {
if (columns != null && !columns.isEmpty()) {
columnsCompleters.put(schemaTable,
new CachedCompleter(new StringsCompleter(columns), ttlInSeconds));
}
}
@ -422,7 +401,7 @@ public class SqlCompleter {
* @return -1 in case of no candidates found, 0 otherwise
*/
private int completeKeyword(String buffer, int cursor, List<CharSequence> candidates) {
return keywordCompleter.complete(buffer, cursor, candidates);
return keywordCompleter.getCompleter().complete(buffer, cursor, candidates);
}
/**
@ -431,7 +410,7 @@ public class SqlCompleter {
* @return -1 in case of no candidates found, 0 otherwise
*/
private int completeSchema(String buffer, int cursor, List<CharSequence> candidates) {
return schemasCompleter.complete(buffer, cursor, candidates);
return schemasCompleter.getCompleter().complete(buffer, cursor, candidates);
}
/**
@ -441,21 +420,12 @@ public class SqlCompleter {
*/
private int completeTable(String schema, String buffer, int cursor,
List<CharSequence> candidates) {
if (schema == null) {
int res = -1;
Set<CharSequence> candidatesSet = new HashSet<>();
for (StringsCompleter stringsCompleter : tablesCompleters.values()) {
int resTable = stringsCompleter.complete(buffer, cursor, candidatesSet);
res = Math.max(res, resTable);
}
candidates.addAll(candidatesSet);
return res;
}
// Wrong schema
if (!tablesCompleters.containsKey(schema) && schema != null)
if (schema == null || !tablesCompleters.containsKey(schema))
return -1;
else
return tablesCompleters.get(schema).complete(buffer, cursor, candidates);
else {
return tablesCompleters.get(schema).getCompleter().complete(buffer, cursor, candidates);
}
}
/**
@ -465,22 +435,12 @@ public class SqlCompleter {
*/
private int completeColumn(String schema, String table, String buffer, int cursor,
List<CharSequence> candidates) {
if (table == null && schema == null) {
int res = -1;
Set<CharSequence> candidatesSet = new HashSet<>();
for (StringsCompleter stringsCompleter : columnsCompleters.values()) {
int resColumn = stringsCompleter.complete(buffer, cursor, candidatesSet);
res = Math.max(res, resColumn);
}
candidates.addAll(candidatesSet);
return res;
}
// Wrong schema or wrong table
if (!tablesCompleters.containsKey(schema) ||
!columnsCompleters.containsKey(schema + "." + table)) {
if (schema == null || table == null || !columnsCompleters.containsKey(schema + "." + table)) {
return -1;
} else {
return columnsCompleters.get(schema + "." + table).complete(buffer, cursor, candidates);
return columnsCompleters.get(schema + "." + table).getCompleter()
.complete(buffer, cursor, candidates);
}
}
@ -489,74 +449,55 @@ public class SqlCompleter {
* a schema, a table of a column or a keyword
*
* @param aliases for every alias contains table name in format schema_name.table_name
* @param isColumnAllowed if false the function will not search and complete columns
* @return -1 in case of no candidates found, 0 otherwise
*/
public int completeName(String buffer, int cursor, List<InterpreterCompletion> candidates,
Map<String, String> aliases, boolean isColumnAllowed) {
Map<String, String> aliases) {
CursorArgument cursorArgument = parseCursorArgument(buffer, cursor);
// points divide the name to the schema, table and column - find them
int pointPos1 = -1;
int pointPos2 = -1;
if (StringUtils.isNotEmpty(buffer)) {
if (buffer.length() > cursor) {
buffer = buffer.substring(0, cursor + 1);
}
pointPos1 = buffer.indexOf('.');
pointPos2 = buffer.indexOf('.', pointPos1 + 1);
}
// find schema and table name if they are
String schema;
String table;
String column;
if (pointPos1 == -1) { // process all
if (cursorArgument.getSchema() == null) { // process all
List<CharSequence> keywordsCandidates = new ArrayList();
List<CharSequence> schemaCandidates = new ArrayList<>();
List<CharSequence> tableCandidates = new ArrayList<>();
List<CharSequence> columnCandidates = new ArrayList<>();
int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates);
int schemaRes = completeSchema(buffer, cursor, schemaCandidates);
int tableRes = completeTable(null, buffer, cursor, tableCandidates);
int columnRes = -1;
if (isColumnAllowed) {
columnRes = completeColumn(null, null, buffer, cursor, columnCandidates);
}
addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name());
addCompletions(candidates, schemaCandidates, CompletionType.schema.name());
addCompletions(candidates, tableCandidates, CompletionType.table.name());
addCompletions(candidates, columnCandidates, CompletionType.column.name());
return NumberUtils.max(new int[]{keywordsRes, schemaRes, tableRes, columnRes});
return NumberUtils.max(new int[]{keywordsRes, schemaRes});
} else {
schema = buffer.substring(0, pointPos1);
schema = cursorArgument.getSchema();
if (aliases.containsKey(schema)) { // process alias case
String alias = aliases.get(schema);
int pointPos = alias.indexOf('.');
schema = alias.substring(0, pointPos);
table = alias.substring(pointPos + 1);
column = buffer.substring(pointPos1 + 1);
} else if (pointPos2 == -1) { // process schema.table case
column = cursorArgument.getColumn();
List<CharSequence> columnCandidates = new ArrayList();
int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(),
columnCandidates);
addCompletions(candidates, columnCandidates, CompletionType.column.name());
// process schema.table case
} else if (cursorArgument.getTable() != null && cursorArgument.getColumn() == null) {
List<CharSequence> tableCandidates = new ArrayList();
table = buffer.substring(pointPos1 + 1);
int tableRes = completeTable(schema, table, cursor - pointPos1 - 1, tableCandidates);
table = cursorArgument.getTable();
int tableRes = completeTable(schema, table, cursorArgument.getCursorPosition(),
tableCandidates);
addCompletions(candidates, tableCandidates, CompletionType.table.name());
return tableRes;
} else {
table = buffer.substring(pointPos1 + 1, pointPos2);
column = buffer.substring(pointPos2 + 1);
List<CharSequence> columnCandidates = new ArrayList();
table = cursorArgument.getTable();
column = cursorArgument.getColumn();
int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(),
columnCandidates);
addCompletions(candidates, columnCandidates, CompletionType.column.name());
}
}
// here in case of column
if (table != null && isColumnAllowed) {
List<CharSequence> columnCandidates = new ArrayList();
int columnRes = completeColumn(schema, table, column, cursor - pointPos2 - 1,
columnCandidates);
addCompletions(candidates, columnCandidates, CompletionType.column.name());
return columnRes;
}
return -1;
}
@ -572,4 +513,92 @@ public class SqlCompleter {
candidate.toString(), meta));
}
}
private CursorArgument parseCursorArgument(String buffer, int cursor) {
CursorArgument result = new CursorArgument();
if (buffer != null && buffer.length() >= cursor) {
String buf = buffer.substring(0, cursor);
if (StringUtils.isNotBlank(buf)) {
ArgumentList argumentList = sqlDelimiter.delimit(buf, cursor);
String cursorArgument = argumentList.getCursorArgument();
if (cursorArgument != null) {
int pointPos1 = cursorArgument.indexOf('.');
int pointPos2 = cursorArgument.indexOf('.', pointPos1 + 1);
if (pointPos1 > -1) {
result.setSchema(cursorArgument.substring(0, pointPos1).trim());
if (pointPos2 > -1) {
result.setTable(cursorArgument.substring(pointPos1 + 1, pointPos2));
result.setColumn(cursorArgument.substring(pointPos2 + 1));
result.setCursorPosition(cursor - pointPos2 - 1);
} else {
result.setTable(cursorArgument.substring(pointPos1 + 1));
result.setCursorPosition(cursor - pointPos1 - 1);
}
}
}
}
}
return result;
}
private class CursorArgument {
private String schema;
private String table;
private String column;
private int cursorPosition;
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getColumn() {
return column;
}
public void setColumn(String column) {
this.column = column;
}
public int getCursorPosition() {
return cursorPosition;
}
public void setCursorPosition(int cursorPosition) {
this.cursorPosition = cursorPosition;
}
public boolean needLoadSchemas() {
if (table == null && column == null) {
return true;
}
return false;
}
public boolean needLoadTables() {
if (schema != null && table != null && column == null) {
return true;
}
return false;
}
public boolean needLoadColumns() {
if (schema != null && table != null && column != null) {
return true;
}
return false;
}
}
}

12
jdbc/src/main/resources/interpreter-setting.json
View file

@ -22,6 +22,12 @@
"defaultValue": "",
"description": "The JDBC user password"
},
"default.completer.ttlInSeconds": {
"envName": null,
"propertyName": "default.completer.ttlInSeconds",
"defaultValue": "120",
"description": "Time to live sql completer in seconds (-1 to update everytime, 0 to disable update)"
},
"default.driver": {
"envName": null,
"propertyName": "default.driver",
@ -40,6 +46,12 @@
"defaultValue": "",
"description": "SQL which executes while opening connection"
},
"default.splitQueries": {
"envName": null,
"propertyName": "default.splitQueries",
"defaultValue": "false",
"description": "Each query is executed apart and returns the result"
},
"common.max_count": {
"envName": null,
"propertyName": "common.max_count",

85
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -27,8 +27,11 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.*;
import java.util.ArrayList;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Properties;
@ -87,7 +90,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
interpreterContext = new InterpreterContext("", "1", null, "", "", new AuthenticationInfo(), null, null, null, null,
interpreterContext = new InterpreterContext("", "1", null, "", "", new AuthenticationInfo("testUser"), null, null, null, null,
null, null);
}
@ -175,21 +178,34 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "insert into test_table(id, name) values ('a', ';\"');" +
"select * from test_table;" +
"select * from test_table WHERE ID = \";'\";" +
"select * from test_table WHERE ID = ';'";
"select * from test_table WHERE ID = ';';" +
"select '\n', ';';" +
"select replace('A\\;B', '\\', 'text');" +
"select '\\', ';';" +
"select '''', ';'";
Properties properties = new Properties();
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
ArrayList<String> multipleSqlArray = t.splitSqlQueries(sqlQuery);
assertEquals(4, multipleSqlArray.size());
List<String> multipleSqlArray = t.splitSqlQueries(sqlQuery);
assertEquals(8, multipleSqlArray.size());
assertEquals("insert into test_table(id, name) values ('a', ';\"')", multipleSqlArray.get(0));
assertEquals("select * from test_table", multipleSqlArray.get(1));
assertEquals("select * from test_table WHERE ID = \";'\"", multipleSqlArray.get(2));
assertEquals("select * from test_table WHERE ID = ';'", multipleSqlArray.get(3));
assertEquals("select '\n', ';'", multipleSqlArray.get(4));
assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(5));
assertEquals("select '\\', ';'", multipleSqlArray.get(6));
assertEquals("select '''', ';'", multipleSqlArray.get(7));
}
@Test
public void testSelectMultipleQuries() throws SQLException, IOException {
public void testQueryWithEsсapedCharacters() throws SQLException, IOException {
String sqlQuery = "select '\\n', ';';" +
"select replace('A\\;B', '\\', 'text');" +
"select '\\', ';';" +
"select '''', ';'";
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
@ -197,6 +213,34 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty("default.splitQueries", "true");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(1).getType());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(2).getType());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(3).getType());
assertEquals("'\\n'\t';'\n\\n\t;\n", interpreterResult.message().get(0).getData());
assertEquals("'Atext;B'\nAtext;B\n", interpreterResult.message().get(1).getData());
assertEquals("'\\'\t';'\n\\\t;\n", interpreterResult.message().get(2).getData());
assertEquals("''''\t';'\n'\t;\n", interpreterResult.message().get(3).getData());
}
@Test
public void testSelectMultipleQueries() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty("default.splitQueries", "true");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
@ -213,6 +257,28 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals("ID\tNAME\n", interpreterResult.message().get(1).getData());
}
@Test
public void testDefaultSplitQuries() throws SQLException, IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table;" +
"select * from test_table WHERE ID = ';';";
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(1, interpreterResult.message().size());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData());
}
@Test
public void testSelectQueryWithNull() throws SQLException, IOException {
Properties properties = new Properties();
@ -296,9 +362,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
jdbcInterpreter.interpret("", interpreterContext);
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3, null);
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3, interpreterContext);
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ", CompletionType.keyword.name());
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select", "select", CompletionType.keyword.name());
assertEquals(1, completionList.size());
assertEquals(true, completionList.contains(correctCompletionKeyword));
@ -465,6 +531,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty("default.splitQueries", "true");
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();

76
jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java
View file

@ -86,7 +86,7 @@ public class SqlCompleterTest {
private void expectedCompletions(String buffer, int cursor,
Set<InterpreterCompletion> expected) {
if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) {
buffer = buffer.substring(0, cursor + 1);
buffer = buffer.substring(0, cursor);
}
List<InterpreterCompletion> candidates = new ArrayList<>();
@ -140,13 +140,10 @@ public class SqlCompleterTest {
private ArgumentCompleter.WhitespaceArgumentDelimiter delimiter =
new ArgumentCompleter.WhitespaceArgumentDelimiter();
private SqlCompleter sqlCompleter = new SqlCompleter();
private SqlCompleter sqlCompleter = new SqlCompleter(0);
@Before
public void beforeTest() throws IOException, SQLException {
Map<String, Set<String>> tables = new HashMap<>();
Map<String, Set<String>> columns = new HashMap<>();
Set<String> schemas = new HashSet<>();
Set<String> keywords = new HashSet<>();
@ -158,37 +155,42 @@ public class SqlCompleterTest {
keywords.add("LIMIT");
keywords.add("FROM");
sqlCompleter.initKeywords(keywords);
schemas.add("prod_dds");
schemas.add("prod_emart");
sqlCompleter.initSchemas(schemas);
Set<String> prod_dds_tables = new HashSet<>();
prod_dds_tables.add("financial_account");
prod_dds_tables.add("customer");
sqlCompleter.initTables("prod_dds", prod_dds_tables);
Set<String> prod_emart_tables = new HashSet<>();
prod_emart_tables.add("financial_account");
tables.put("prod_dds", prod_dds_tables);
tables.put("prod_emart", prod_emart_tables);
sqlCompleter.initTables("prod_emart", prod_emart_tables);
Set<String> prod_dds_financial_account_columns = new HashSet<>();
prod_dds_financial_account_columns.add("account_rk");
prod_dds_financial_account_columns.add("account_id");
sqlCompleter.initColumns("prod_dds.financial_account", prod_dds_financial_account_columns);
Set<String> prod_dds_customer_columns = new HashSet<>();
prod_dds_customer_columns.add("customer_rk");
prod_dds_customer_columns.add("name");
prod_dds_customer_columns.add("birth_dt");
sqlCompleter.initColumns("prod_dds.customer", prod_dds_customer_columns);
Set<String> prod_emart_financial_account_columns = new HashSet<>();
prod_emart_financial_account_columns.add("account_rk");
prod_emart_financial_account_columns.add("balance_amt");
columns.put("prod_dds.financial_account", prod_dds_financial_account_columns);
columns.put("prod_dds.customer", prod_dds_customer_columns);
columns.put("prod_emart.financial_account", prod_emart_financial_account_columns);
sqlCompleter.init(schemas, tables, columns, keywords);
sqlCompleter.initColumns("prod_emart.financial_account", prod_emart_financial_account_columns);
tester = new CompleterTester(sqlCompleter);
}
@ -223,8 +225,8 @@ public class SqlCompleterTest {
int cursor = 0;
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
assertEquals(17, candidates.size());
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(9, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name())));
@ -234,14 +236,6 @@ public class SqlCompleterTest {
assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
assertTrue(candidates.contains(new InterpreterCompletion("customer", "customer", CompletionType.table.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("name", "name", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())));
}
@Test
@ -250,7 +244,7 @@ public class SqlCompleterTest {
int cursor = 3;
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
@ -262,7 +256,7 @@ public class SqlCompleterTest {
int cursor = 11;
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(1, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
}
@ -273,7 +267,7 @@ public class SqlCompleterTest {
int cursor = 30;
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
@ -286,7 +280,7 @@ public class SqlCompleterTest {
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
aliases.put("a", "prod_dds.financial_account");
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
@ -299,7 +293,7 @@ public class SqlCompleterTest {
List<InterpreterCompletion> candidates = new ArrayList<>();
Map<String, String> aliases = new HashMap<>();
aliases.put("a", "prod_dds.financial_account");
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
@ -308,14 +302,14 @@ public class SqlCompleterTest {
@Test
public void testSchemaAndTable() {
String buffer = "select * from prod_emart.fi";
tester.buffer(buffer).from(19).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart ", "prod_emart ", CompletionType.schema.name()))).test();
tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account ", "financial_account ", CompletionType.table.name()))).test();
tester.buffer(buffer).from(20).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))).test();
tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))).test();
}
@Test
public void testEdges() {
String buffer = " ORDER ";
tester.buffer(buffer).from(2).to(6).expect(newHashSet(new InterpreterCompletion("ORDER ", "ORDER ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(3).to(7).expect(newHashSet(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(0).to(1).expect(newHashSet(
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
@ -325,37 +319,29 @@ public class SqlCompleterTest {
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()),
new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()),
new InterpreterCompletion("customer", "customer", CompletionType.table.name()),
new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()),
new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()),
new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()),
new InterpreterCompletion("name", "name", CompletionType.column.name()),
new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()),
new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())
)).test();
}
@Test
public void testMultipleWords() {
String buffer = "SELE FRO LIM";
tester.buffer(buffer).from(1).to(3).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(6).to(7).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(9).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT ", "LIMIT ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(2).to(4).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(6).to(8).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(10).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))).test();
}
@Test
public void testMultiLineBuffer() {
String buffer = " \n SELE\nFRO";
tester.buffer(buffer).from(4).to(6).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(5).to(7).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test();
}
@Test
public void testMultipleCompletionSuggestions() {
String buffer = "SU";
tester.buffer(buffer).from(1).to(2).expect(newHashSet(
tester.buffer(buffer).from(2).to(2).expect(newHashSet(
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))

18
livy/pom.xml
View file

@ -37,14 +37,10 @@
<properties>
<!--library versions-->
<commons.exec.version>1.3</commons.exec.version>
<httpcomponents.client.version>4.3.4</httpcomponents.client.version>
<spring.web.version>4.3.0.RELEASE</spring.web.version>
<spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
<!--test library versions-->
<achilles.version>3.2.4-Zeppelin</achilles.version>
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
<livy.version>0.3.0</livy.version>
<spark.version>2.1.0</spark.version>
<hadoop.version>2.6.0</hadoop.version>
@ -80,7 +76,6 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponents.client.version}</version>
</dependency>
<dependency>
@ -106,19 +101,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.cloudera.livy</groupId>
<artifactId>livy-integration-test</artifactId>

83
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
View file

@ -21,9 +21,20 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.StringUtils;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.interpreter.*;
@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.SSLContext;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyStore;
import java.security.Principal;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -52,7 +63,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* Base class for livy interpreters.
*/
@ -331,7 +341,17 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo,
boolean displayAppInfo) {
if (stmtInfo.output != null && stmtInfo.output.isError()) {
return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue);
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR);
StringBuilder sb = new StringBuilder();
sb.append(stmtInfo.output.evalue);
// in case evalue doesn't have newline char
if (!stmtInfo.output.evalue.contains("\n"))
sb.append("\n");
if (stmtInfo.output.traceback != null) {
sb.append(StringUtils.join(stmtInfo.output.traceback));
}
result.add(sb.toString());
return result;
} else if (stmtInfo.isCancelled()) {
// corner case, output might be null if it is cancelled.
return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled");
@ -407,6 +427,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private RestTemplate createRestTemplate() {
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
StringUtils.isNotEmpty(principal);
HttpClient httpClient = null;
if (livyURL.startsWith("https:")) {
String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
@ -427,7 +452,37 @@ public abstract class BaseLivyInterpreter extends Interpreter {
.loadTrustMaterial(trustStore)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
httpClient = HttpClients.custom().setSSLSocketFactory(csf).build();
HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf);
RequestConfig reqConfig = new RequestConfig() {
@Override
public boolean isAuthenticationEnabled() {
return true;
}
};
httpClientBuilder.setDefaultRequestConfig(reqConfig);
Credentials credentials = new Credentials() {
@Override
public String getPassword() {
return null;
}
@Override
public Principal getUserPrincipal() {
return null;
}
};
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, credentials);
httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
if (isSpnegoEnabled) {
Registry<AuthSchemeProvider> authSchemeProviderRegistry =
RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
.build();
httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry);
}
httpClient = httpClientBuilder.build();
} catch (Exception e) {
throw new RuntimeException("Failed to create SSL HttpClient", e);
} finally {
@ -441,9 +496,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
if (isSpnegoEnabled) {
if (httpClient == null) {
return new KerberosRestTemplate(keytabLocation, principal);
} else {
@ -615,7 +669,18 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
public static StatementInfo fromJson(String json) {
return gson.fromJson(json, StatementInfo.class);
String right_json = "";
try {
gson.fromJson(json, StatementInfo.class);
right_json = json;
} catch (Exception e) {
if (json.contains("\"traceback\":{}")) {
LOGGER.debug("traceback type mismatch, replacing the mismatching part ");
right_json = json.replace("\"traceback\":{}", "\"traceback\":[]");
LOGGER.debug("new json string is {}", right_json);
}
}
return gson.fromJson(right_json, StatementInfo.class);
}
public boolean isAvailable() {
@ -632,7 +697,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
public Data data;
public String ename;
public String evalue;
public Object traceback;
public String[] traceback;
public TableMagic tableMagic;
public boolean isError() {

9
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -239,6 +239,15 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
this.sparkInterpreter.close();
}
@Override
public int getProgress(InterpreterContext context) {
if (this.sparkInterpreter != null) {
return this.sparkInterpreter.getProgress(context);
} else {
return 0;
}
}
@Override
protected String extractAppId() throws LivyException {
// it wont' be called because it would delegate to LivySparkInterpreter

23
livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
View file

@ -308,6 +308,8 @@ public class LivyInterpreterIT {
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertTrue(result.message().get(0).getData().contains("tableName"));
int r = sqlInterpreter.getProgress(context);
assertTrue(r == 0);
} finally {
sqlInterpreter.close();
}
@ -521,7 +523,7 @@ public class LivyInterpreterIT {
}
@Test
public void testPySparkInterpreter() {
public void testPySparkInterpreter() throws LivyException {
if (!checkPreCondition()) {
return;
}
@ -534,6 +536,24 @@ public class LivyInterpreterIT {
"title", "text", authInfo, null, null, null, null, null, output);
pysparkInterpreter.open();
// test traceback msg
try {
pysparkInterpreter.getLivyVersion();
// for livy version >=0.3 , input some erroneous spark code, check the shown result is more than one line
InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().split("\n").length>1);
assertTrue(result.message().get(0).getData().contains("Traceback"));
} catch (APINotFoundException e) {
// only livy 0.2 can throw this exception since it doesn't have /version endpoint
// in livy 0.2, most error msg is encapsulated in evalue field, only print(a) in pyspark would return none-empty
// traceback
InterpreterResult result = pysparkInterpreter.interpret("print(a)", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().split("\n").length>1);
assertTrue(result.message().get(0).getData().contains("Traceback"));
}
try {
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@ -761,6 +781,7 @@ public class LivyInterpreterIT {
}
}
private boolean isSpark2(BaseLivyInterpreter interpreter, InterpreterContext context) {
InterpreterResult result = null;
if (interpreter instanceof LivySparkRInterpreter) {

2
notebook/2A94M5J1Z/note.json
View file

@ -83,7 +83,7 @@
"msg": [
{
"type": "TEXT",
"data": "\nimport org.apache.commons.io.IOUtils\n\nimport java.net.URL\n\nimport java.nio.charset.Charset\n\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[0] at parallelize at \u003cconsole\u003e:32\n\ndefined class Bank\n\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\n\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n"
"data": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[36] at parallelize at \u003cconsole\u003e:43\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n"
}
]
},

18
pom.xml
View file

@ -78,6 +78,7 @@
<module>scio</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-jupyter</module>
<module>zeppelin-distribution</module>
</modules>
@ -96,8 +97,8 @@
<gson-extras.version>0.2.1</gson-extras.version>
<guava.version>15.0</guava.version>
<jetty.version>9.2.15.v20160210</jetty.version>
<httpcomponents.core.version>4.3.3</httpcomponents.core.version>
<httpcomponents.client.version>4.3.6</httpcomponents.client.version>
<httpcomponents.core.version>4.4.1</httpcomponents.core.version>
<httpcomponents.client.version>4.5.1</httpcomponents.client.version>
<httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
<commons.lang.version>2.5</commons.lang.version>
<commons.configuration.version>1.9</commons.configuration.version>
@ -105,6 +106,7 @@
<commons.io.version>2.4</commons.io.version>
<commons.collections.version>3.2.1</commons.collections.version>
<commons.logging.version>1.1.1</commons.logging.version>
<commons.cli.version>1.3.1</commons.cli.version>
<shiro.version>1.3.2</shiro.version>
<!-- test library versions -->
@ -134,6 +136,9 @@
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
<!-- to be able to exclude some tests using command line -->
<tests.to.exclude/>
</properties>
<dependencyManagement>
@ -229,6 +234,12 @@
<version>${commons.logging.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commons.cli.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@ -555,6 +566,9 @@
<version>${plugin.surefire.version}</version>
<configuration combine.children="append">
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
<excludes>
<exclude>${tests.to.exclude}</exclude>
</excludes>
</configuration>
<!-- <excludes> <exclude>**/itest/**</exclude> </excludes> <executions>
<execution> <id>surefire-itest</id> <phase>integration-test</phase> <goals>

1
r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala
View file

@ -23,7 +23,6 @@ import java.util._
import org.apache.commons.codec.binary.{Base64, StringUtils}
import org.apache.zeppelin.interpreter.Interpreter.FormType
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter
import org.apache.zeppelin.interpreter.{InterpreterContext, _}
import org.apache.zeppelin.scheduler.Scheduler
import org.apache.zeppelin.spark.SparkInterpreter

8
scripts/docker/spark-cluster-managers/spark_mesos/Dockerfile
View file

@ -14,10 +14,10 @@
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV HADOOP_PROFILE 2.3
ENV HADOOP_VERSION 2.3.0
ENV SPARK_PROFILE 2.1
ENV SPARK_VERSION 2.1.1
ENV HADOOP_PROFILE 2.7
ENV HADOOP_VERSION 2.7.0
# Update the image with the latest packages
RUN yum update -y; yum clean all

6
scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile
View file

@ -14,9 +14,9 @@
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 1.6
ENV SPARK_VERSION 1.6.2
ENV HADOOP_PROFILE 2.3
ENV SPARK_PROFILE 2.1
ENV SPARK_VERSION 2.1.1
ENV HADOOP_PROFILE 2.7
ENV SPARK_HOME /usr/local/spark
# Update the image with the latest packages

4
scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile
View file

@ -14,8 +14,8 @@
# limitations under the License.
FROM centos:centos6
ENV SPARK_PROFILE 2.0
ENV SPARK_VERSION 2.0.0
ENV SPARK_PROFILE 2.1
ENV SPARK_VERSION 2.1.1
ENV HADOOP_PROFILE 2.7
ENV HADOOP_VERSION 2.7.0

14
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -276,10 +276,13 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
public class PythonInterpretRequest {
public String statements;
public String jobGroup;
public String jobDescription;
public PythonInterpretRequest(String statements, String jobGroup) {
public PythonInterpretRequest(String statements, String jobGroup,
String jobDescription) {
this.statements = statements;
this.jobGroup = jobGroup;
this.jobDescription = jobDescription;
}
public String statements() {
@ -289,6 +292,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
public String jobGroup() {
return jobGroup;
}
public String jobDescription() {
return jobDescription;
}
}
Integer statementSetNotifier = new Integer(0);
@ -395,10 +402,11 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return new InterpreterResult(Code.ERROR, errorMessage);
}
String jobGroup = Utils.buildJobGroupId(context);
String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo());
SparkZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
__zeppelin__.setInterpreterContext(context);
__zeppelin__.setGui(context.getGui());
pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup);
pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup, jobDesc);
statementOutput = null;
synchronized (statementSetNotifier) {
@ -476,7 +484,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return new LinkedList<>();
}
pythonInterpretRequest = new PythonInterpretRequest(completionCommand, "");
pythonInterpretRequest = new PythonInterpretRequest(completionCommand, "", "");
statementOutput = null;
synchronized (statementSetNotifier) {

6
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -36,7 +36,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkEnv;
import org.apache.spark.SecurityManager;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.repl.SparkILoop;
@ -58,6 +57,7 @@ import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -171,6 +171,7 @@ public class SparkInterpreter extends Interpreter {
String jobUrl = getJobUrl(jobId);
String noteId = Utils.getNoteId(jobGroupId);
String paragraphId = Utils.getParagraphId(jobGroupId);
if (jobUrl != null && noteId != null && paragraphId != null) {
RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient();
Map<String, String> infos = new java.util.HashMap<>();
@ -1182,7 +1183,8 @@ public class SparkInterpreter extends Interpreter {
public InterpreterResult interpret(String[] lines, InterpreterContext context) {
synchronized (this) {
z.setGui(context.getGui());
sc.setJobGroup(Utils.buildJobGroupId(context), "Zeppelin", false);
String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo());
sc.setJobGroup(Utils.buildJobGroupId(context), jobDesc, false);
InterpreterResult r = interpretInput(lines, context);
sc.clearJobGroup();
return r;

9
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -21,6 +21,7 @@ import static org.apache.zeppelin.spark.ZeppelinRDisplay.render;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkRBackend;
import org.apache.spark.api.java.JavaSparkContext;
@ -114,7 +115,9 @@ public class SparkRInterpreter extends Interpreter {
}
String jobGroup = Utils.buildJobGroupId(interpreterContext);
sparkInterpreter.getSparkContext().setJobGroup(jobGroup, "Zeppelin", false);
String jobDesc = "Started by: " +
Utils.getUserName(interpreterContext.getAuthenticationInfo());
sparkInterpreter.getSparkContext().setJobGroup(jobGroup, jobDesc, false);
String imageWidth = getProperty("zeppelin.R.image.width");
@ -139,10 +142,10 @@ public class SparkRInterpreter extends Interpreter {
// assign setJobGroup to dummy__, otherwise it would print NULL for this statement
if (Utils.isSpark2()) {
setJobGroup = "dummy__ <- setJobGroup(\"" + jobGroup +
"\", \"zeppelin sparkR job group description\", TRUE)";
"\", \" +" + jobDesc + "\", TRUE)";
} else if (getSparkInterpreter().getSparkVersion().newerThanEquals(SparkVersion.SPARK_1_5_0)) {
setJobGroup = "dummy__ <- setJobGroup(sc, \"" + jobGroup +
"\", \"zeppelin sparkR job group description\", TRUE)";
"\", \"" + jobDesc + "\", TRUE)";
}
logger.debug("set JobGroup:" + setJobGroup);
lines = setJobGroup + "\n" + lines;

3
spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
View file

@ -105,7 +105,8 @@ public class SparkSqlInterpreter extends Interpreter {
sc.setLocalProperty("spark.scheduler.pool", null);
}
sc.setJobGroup(Utils.buildJobGroupId(context), "Zeppelin", false);
String jobDesc = "Started by: " + Utils.getUserName(context.getAuthenticationInfo());
sc.setJobGroup(Utils.buildJobGroupId(context), jobDesc, false);
Object rdd = null;
try {
// method signature of sqlc.sql() is changed

12
spark/src/main/java/org/apache/zeppelin/spark/Utils.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.spark;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -123,4 +124,15 @@ class Utils {
int secondIndex = jobgroupId.indexOf("-", indexOf + 1);
return jobgroupId.substring(secondIndex + 1, jobgroupId.length());
}
public static String getUserName(AuthenticationInfo info) {
String uName = "";
if (info != null) {
uName = info.getUser();
}
if (uName == null || uName.isEmpty()) {
uName = "anonymous";
}
return uName;
}
}

3
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -298,6 +298,7 @@ while True :
try:
stmts = req.statements().split("\n")
jobGroup = req.jobGroup()
jobDesc = req.jobDescription()
# Get post-execute hooks
try:
@ -318,7 +319,7 @@ while True :
if stmts:
# use exec mode to compile the statements except the last statement,
# so that the last statement's evaluation will be printed to stdout
sc.setJobGroup(jobGroup, "Zeppelin")
sc.setJobGroup(jobGroup, jobDesc)
code = compile('\n'.join(stmts), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
to_run_hooks = []
if (nhooks > 0):

14
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -32,7 +32,6 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Avro (org.apache.avro:avro:1.7.7 - http://avro.apache.org)
(Apache 2.0) Apache Curator (org.apache.curator:curator:2.4.0 - http://curator.apache.org/)
(Apache 2.0) Apache Cassandra (http://cassandra.apache.org/)
(Apache 2.0) Apache CXF (http://cxf.apache.org/)
(Apache 2.0) Apache HBase (http://hbase.apache.org/)
(Apache 2.0) Apache Ignite (http://ignite.apache.org/)
(Apache 2.0) Apache Kylin (http://kylin.apache.org/)
@ -118,6 +117,7 @@ The following components are provided under Apache License.
(Apache 2.0) Utility classes for Jetty (org.mortbay.jetty:jetty-util:6.1.26 - http://javadox.com/org.mortbay.jetty/jetty/6.1.26/overview-tree.html)
(Apache 2.0) Servlet API (org.mortbay.jetty:servlet-api:2.5-20081211 - https://en.wikipedia.org/wiki/Jetty_(web_server))
(Apache 2.0) Google HTTP Client Library for Java (com.google.http-client:google-http-client-jackson2:1.21.0 - https://github.com/google/google-http-java-client/tree/dev/google-http-client-jackson2)
(Apache 2.0) validation-api (javax.validation - http://beanvalidation.org/)
(Apache 2.0) pegdown (org.pegdown:pegdown:1.6.0 - https://github.com/sirthias/pegdown)
(Apache 2.0) parboiled-java (org.parboiled:parboiled-java:1.1.7 - https://github.com/sirthias/parboiled)
(Apache 2.0) parboiled-core (org.parboiled:parboiled-core:1.1.7 - https://github.com/sirthias/parboiled)
@ -244,9 +244,9 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) lodash v3.9.3 (https://lodash.com/) - https://github.com/lodash/lodash/blob/3.9.3/LICENSE.txt
(The MIT License) angular-filter v0.5.4 (https://github.com/a8m/angular-filter) - https://github.com/a8m/angular-filter/blob/v0.5.4/license.md
(The MIT License) ngToast v2.0.0 (http://tamerayd.in/ngToast/) - http://tameraydin.mit-license.org/
(The MIT License) Handsontable v0.24.2 (https://github.com/handsontable/handsontable) - https://github.com/handsontable/handsontable/blob/master/LICENSE
(The MIT License) Zeroclipboard v2.2.0 (https://github.com/zeroclipboard/zeroclipboard) - https://github.com/zeroclipboard/zeroclipboard/blob/v2.2.0/LICENSE
(The MIT License) Moment v2.9.0 (https://github.com/moment/moment) - https://github.com/moment/moment/blob/2.9.0/LICENSE
(The MIT License) moment v2.18.1 (https://github.com/moment/moment) - https://github.com/moment/moment/blob/2.18.1/LICENSE
(The MIT License) moment-duration-format v1.3.0 (https://github.com/jsmreese/moment-duration-format) - https://github.com/jsmreese/moment-duration-format/blob/1.3.0/LICENSE
(The MIT License) angular-ui-grid v4.0.4 (https://github.com/angular-ui/ui-grid) - https://github.com/angular-ui/ui-grid/blob/v4.0.4/LICENSE.md
(The MIT License) Pikaday v1.3.2 (https://github.com/dbushell/Pikaday) - https://github.com/dbushell/Pikaday/blob/1.3.2/LICENSE
(The MIT License) slf4j v1.7.10 (org.slf4j:slf4j-api:jar:1.7.10 - http://www.slf4j.org) - http://www.slf4j.org/license.html
(The MIT License) slf4j v1.7.21 (org.slf4j:slf4j-simple:1.7.21 - http://www.slf4j.org) - http://www.slf4j.org/license.html
@ -271,6 +271,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
(The MIT License) Unirest 1.4.9 (com.mashape.unirest:unirest-java:1.4.9 - https://github.com/Mashape/unirest-java)
(The MIT License) ngclipboard v1.1.1 (https://github.com/sachinchoolur/ngclipboard) - https://github.com/sachinchoolur/ngclipboard/blob/1.1.1/LICENSE
(The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE
(The MIT License) angular-viewport-watch 0.135 (https://github.com/wix/angular-viewport-watch) - https://github.com/wix/angular-viewport-watch/blob/master/LICENSE
========================================================================
BSD-style licenses
@ -348,9 +349,10 @@ The following components are provided under the CDDL License.
(CDDL 1.0) javax.activation (javax.activation:activation:jar:1.1.1 - http://java.sun.com/javase/technologies/desktop/javabeans/jaf/index.jsp)
(CDDL 1.0) java.annotation (javax.annotation:javax.annotation-api:jar:1.2:compile - http://jcp.org/en/jsr/detail?id=250)
(CDDL 1.1) Jersey (com.sun.jersey:jersey:jar:1.9 - https://jersey.java.net/)
(CDDL 1.1) jersey-core (org.glassfish.jersey.core:jersey-core:2.22.2 - https://jersey.java.net/)
(CDDL 1.1) hk2 (org.glassfish.hk2 - https://hk2.java.net/2.5.0-b03/)
(CDDL 1.0) javax.ws.rs-api (javax.ws.rs - https://jax-rs-spec.java.net/)
(CDDL 1.1) jersey-client (org.glassfish.jersey.client:jersey-core:2.22.2 - https://jersey.java.net/)
(CDDL 1.1) jersey-media (org.glassfish.jersey.media:jersey-core:2.22.2 - https://jersey.java.net/)
(CDDL 1.1) jersey-core (com.sun.jersey:jersey-core:1.9 - https://jersey.java.net/jersey-core/)
(CDDL 1.1) jersey-json (com.sun.jersey:jersey-json:1.9 - https://jersey.java.net/jersey-json/)
(CDDL 1.1) jersey-server (com.sun.jersey:jersey-server:1.9 - https://jersey.java.net/jersey-server/)

130
zeppelin-distribution/src/bin_license/licenses/LICENSE-jersey-2 Normal file
View file

@ -0,0 +1,130 @@
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1)
1. Definitions.
1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications.
1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor.
1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof.
1.4. “Executable” means the Covered Software in any form other than Source Code.
1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License.
1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License.
1.7. “License” means this document.
1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
1.9. “Modifications” means the Source Code and Executable form of any of the following:
A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications;
B. Any new file that contains any part of the Original Software or previous Modification; or
C. Any new file that is contributed or otherwise made available under the terms of this License.
1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License.
1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor.
1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code.
1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity.
2. License Grants.
2.1. The Initial Developer Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof).
(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License.
(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices.
2.2. Contributor Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination).
(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party.
(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor.
3. Distribution Obligations.
3.1. Availability of Source Code.
Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange.
3.2. Modifications.
The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License.
3.3. Required Notices.
You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer.
3.4. Application of Additional Terms.
You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer.
3.5. Distribution of Executable Versions.
You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer.
3.6. Larger Works.
You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software.
4. Versions of the License.
4.1. New Versions.
Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License.
4.2. Effect of New Versions.
You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward.
4.3. Modified Versions.
When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License.
5. DISCLAIMER OF WARRANTY.
COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
6. TERMINATION.
6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive.
6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant.
6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license.
6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination.
7. LIMITATION OF LIABILITY.
UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
8. U.S. GOVERNMENT END USERS.
The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License.
9. MISCELLANEOUS.
This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software.
10. RESPONSIBILITY FOR CLAIMS.
As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California

44
zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/CachedCompleter.java Normal file
View file

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.completer;
import jline.console.completer.Completer;
/**
* Completer with time to live
*/
public class CachedCompleter {
private Completer completer;
private int ttlInSeconds;
private long createdAt;
public CachedCompleter(Completer completer, int ttlInSeconds) {
this.completer = completer;
this.ttlInSeconds = ttlInSeconds;
this.createdAt = System.currentTimeMillis();
}
public boolean isExpired() {
if (ttlInSeconds == -1 || (ttlInSeconds > 0 &&
(System.currentTimeMillis() - createdAt) / 1000 > ttlInSeconds)) {
return true;
}
return false;
}
public Completer getCompleter() {
return completer;
}
}

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/completer/StringsCompleter.java
View file

@ -61,8 +61,9 @@ public class StringsCompleter implements Completer {
if (buffer == null) {
candidates.addAll(strings);
} else {
String bufferTmp = buffer.toUpperCase();
for (String match : strings.tailSet(buffer)) {
String part = buffer.substring(0, cursor);
String bufferTmp = part.toUpperCase();
for (String match : strings.tailSet(part)) {
String matchTmp = match.toUpperCase();
if (!matchTmp.startsWith(bufferTmp)) {
break;

3
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java
View file

@ -50,7 +50,8 @@ public class InterpreterResult implements Serializable {
TABLE,
IMG,
SVG,
NULL
NULL,
NETWORK
}
Code code;

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java
View file

@ -477,15 +477,18 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
/**
* Wait for eventQueue becomes empty
*/
public void waitForEventQueueBecomesEmpty() {
public void waitForEventQueueBecomesEmpty(long atMost) {
long startTime = System.currentTimeMillis();
synchronized (eventQueue) {
while (!eventQueue.isEmpty()) {
while (!eventQueue.isEmpty() && (System.currentTimeMillis() - startTime) < atMost) {
try {
eventQueue.wait(100);
} catch (InterruptedException e) {
// ignore exception
}
}
if (!eventQueue.isEmpty())
eventQueue.clear();
}
}
}

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -80,6 +80,8 @@ public class RemoteInterpreterServer
private Map<String, Object> remoteWorksResponsePool;
private ZeppelinRemoteWorksController remoteWorksController;
private final long DEFAULT_SHUTDOWN_TIMEOUT = 2000;
public RemoteInterpreterServer(int port) throws TTransportException {
this.port = port;
@ -99,7 +101,7 @@ public class RemoteInterpreterServer
@Override
public void shutdown() throws TException {
eventClient.waitForEventQueueBecomesEmpty();
eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
if (interpreterGroup != null) {
interpreterGroup.close();
}
@ -111,7 +113,8 @@ public class RemoteInterpreterServer
// this case, need to force kill the process
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 2000 && server.isServing()) {
while (System.currentTimeMillis() - startTime < DEFAULT_SHUTDOWN_TIMEOUT &&
server.isServing()) {
try {
Thread.sleep(300);
} catch (InterruptedException e) {

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -259,10 +259,18 @@ public abstract class Job {
return dateStarted;
}
public synchronized void setDateStarted(Date startedAt) {
dateStarted = startedAt;
}
public synchronized Date getDateFinished() {
return dateFinished;
}
public synchronized void setDateFinished(Date finishedAt) {
dateFinished = finishedAt;
}
public abstract void setResult(Object results);
public synchronized String getErrorMessage() {

63
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java
View file

@ -20,6 +20,9 @@ package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer;
@ -72,5 +75,65 @@ public class RemoteInterpreterServerTest {
assertEquals(false, running);
}
class ShutdownRun implements Runnable {
private RemoteInterpreterServer serv = null;
public ShutdownRun(RemoteInterpreterServer serv) {
this.serv = serv;
}
@Override
public void run() {
try {
serv.shutdown();
} catch (Exception ex) {};
}
};
@Test
public void testStartStopWithQueuedEvents() throws InterruptedException, IOException, TException {
RemoteInterpreterServer server = new RemoteInterpreterServer(
RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces());
assertEquals(false, server.isRunning());
server.start();
long startTime = System.currentTimeMillis();
boolean running = false;
while (System.currentTimeMillis() - startTime < 10 * 1000) {
if (server.isRunning()) {
running = true;
break;
} else {
Thread.sleep(200);
}
}
assertEquals(true, running);
assertEquals(true, RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", server.getPort()));
//just send an event on the client queue
server.eventClient.onAppStatusUpdate("","","","");
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
Runnable task = new ShutdownRun(server);
executor.schedule(task, 0, TimeUnit.MILLISECONDS);
while (System.currentTimeMillis() - startTime < 10 * 1000) {
if (server.isRunning()) {
Thread.sleep(200);
} else {
running = false;
break;
}
}
executor.shutdown();
//cleanup environment for next tests
server.shutdown();
assertEquals(false, running);
}
}

64
zeppelin-jupyter/pom.xml Normal file
View file

@ -0,0 +1,64 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<artifactId>zeppelin-jupyter</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Jupyter Support</name>
<description>Jupyter support for Apache Zeppelin</description>
<dependencies>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>org.danilopianini</groupId>
<artifactId>gson-extras</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<!-- Test -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
</dependencies>
</project>

204
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java Normal file
View file

@ -0,0 +1,204 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.google.common.base.Joiner;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.typeadapters.RuntimeTypeAdapterFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.zeppelin.jupyter.nbformat.Cell;
import org.apache.zeppelin.jupyter.nbformat.CodeCell;
import org.apache.zeppelin.jupyter.nbformat.DisplayData;
import org.apache.zeppelin.jupyter.nbformat.Error;
import org.apache.zeppelin.jupyter.nbformat.ExecuteResult;
import org.apache.zeppelin.jupyter.nbformat.HeadingCell;
import org.apache.zeppelin.jupyter.nbformat.MarkdownCell;
import org.apache.zeppelin.jupyter.nbformat.Nbformat;
import org.apache.zeppelin.jupyter.nbformat.Output;
import org.apache.zeppelin.jupyter.nbformat.RawCell;
import org.apache.zeppelin.jupyter.nbformat.Stream;
import org.apache.zeppelin.jupyter.zformat.Note;
import org.apache.zeppelin.jupyter.zformat.Paragraph;
import org.apache.zeppelin.jupyter.zformat.Result;
import org.apache.zeppelin.jupyter.zformat.TypeData;
/**
*
*/
public class JupyterUtil {
private static final String TEXT_PLAIN = "text/plain";
private static final String IMAGE_PNG = "image/png";
private final RuntimeTypeAdapterFactory<Cell> cellTypeFactory;
private final RuntimeTypeAdapterFactory<Output> outputTypeFactory;
public JupyterUtil() {
this.cellTypeFactory = RuntimeTypeAdapterFactory.of(Cell.class, "cell_type")
.registerSubtype(MarkdownCell.class, "markdown").registerSubtype(CodeCell.class, "code")
.registerSubtype(RawCell.class, "raw").registerSubtype(HeadingCell.class, "heading");
this.outputTypeFactory = RuntimeTypeAdapterFactory.of(Output.class, "output_type")
.registerSubtype(ExecuteResult.class, "execute_result")
.registerSubtype(DisplayData.class, "display_data").registerSubtype(Stream.class, "stream")
.registerSubtype(Error.class, "error");
}
public Nbformat getNbformat(Reader in) {
return getNbformat(in, new GsonBuilder());
}
public Nbformat getNbformat(Reader in, GsonBuilder gsonBuilder) {
return getGson(gsonBuilder).fromJson(in, Nbformat.class);
}
public Note getNote(Reader in, String codeReplaced, String markdownReplaced) {
return getNote(in, new GsonBuilder(), codeReplaced, markdownReplaced);
}
public Note getNote(Reader in, GsonBuilder gsonBuilder, String codeReplaced,
String markdownReplaced) {
return getNote(getNbformat(in, gsonBuilder), codeReplaced, markdownReplaced);
}
public Note getNote(Nbformat nbformat, String codeReplaced, String markdownReplaced) {
Note note = new Note();
String name = nbformat.getMetadata().getTitle();
if (null == name) {
name = "Note converted from Jupyter";
}
note.setName(name);
String lineSeparator = System.lineSeparator();
Paragraph paragraph;
List<Paragraph> paragraphs = new ArrayList<>();
String interpreterName;
List<TypeData> typeDataList;
String type;
String result;
for (Cell cell : nbformat.getCells()) {
paragraph = new Paragraph();
typeDataList = new ArrayList<>();
if (cell instanceof CodeCell) {
interpreterName = codeReplaced;
for (Output output : ((CodeCell) cell).getOutputs()) {
TypeData typeData;
if (output instanceof Stream) {
type = TypeData.TEXT;
result = Joiner.on(lineSeparator).join(((Stream) output).getText());
typeData = new TypeData(type, result);
typeDataList.add(typeData);
} else if (output instanceof ExecuteResult || output instanceof DisplayData) {
Map<String, Object> data = (output instanceof ExecuteResult) ?
((ExecuteResult) output).getData() :
((DisplayData) output).getData();
for (Map.Entry<String, Object> datum : data.entrySet()) {
if (TEXT_PLAIN.equals(datum.getKey())) {
type = TypeData.TEXT;
result = Joiner.on(lineSeparator).join((List<String>) datum.getValue());
} else if (IMAGE_PNG.equals(datum.getKey())) {
type = TypeData.HTML;
result = makeHTML(((String) datum.getValue()).replace("\n", ""));
} else {
type = TypeData.TEXT;
result = datum.getValue().toString();
}
typeData = new TypeData(type, result);
typeDataList.add(typeData);
}
} else {
// Error
Error error = (Error) output;
type = TypeData.TEXT;
result =
Joiner.on(lineSeparator).join(new String[] {error.getEname(), error.getEvalue()});
typeData = new TypeData(type, result);
typeDataList.add(typeData);
}
}
} else if (cell instanceof MarkdownCell || cell instanceof HeadingCell) {
interpreterName = markdownReplaced;
} else {
interpreterName = "";
}
paragraph.setText(
interpreterName + lineSeparator + Joiner.on(lineSeparator).join(cell.getSource()));
paragraph.setResults(new Result(Result.SUCCESS, typeDataList));
paragraphs.add(paragraph);
}
note.setParagraphs(paragraphs);
return note;
}
private Gson getGson(GsonBuilder gsonBuilder) {
return gsonBuilder.registerTypeAdapterFactory(cellTypeFactory)
.registerTypeAdapterFactory(outputTypeFactory).create();
}
private String makeHTML(String image) {
return "<div style='width:auto;height:auto'><img src=data:image/png;base64," + image
+ " style='width=auto;height:auto'/></div>";
}
public static void main(String[] args) throws ParseException, IOException {
Options options = new Options();
options.addOption("i", true, "Jupyter notebook file");
options.addOption("o", true, "Zeppelin note file. Default: note.json");
CommandLineParser parser = new DefaultParser();
CommandLine cmd = parser.parse(options, args);
if (!cmd.hasOption("i")) {
new HelpFormatter().printHelp("java " + JupyterUtil.class.getName(), options);
System.exit(1);
}
Path jupyterPath = Paths.get(cmd.getOptionValue("i"));
Path zeppelinPath = Paths.get(cmd.hasOption("o") ? cmd.getOptionValue("o") : "note.json");
try (BufferedReader in = new BufferedReader(new FileReader(jupyterPath.toFile()));
FileWriter fw = new FileWriter(zeppelinPath.toFile())) {
Note note = new JupyterUtil().getNote(in, "python", "md");
Gson gson = new GsonBuilder().setPrettyPrinting().create();
gson.toJson(note, fw);
}
}
}

29
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Author.java Normal file
View file

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class Author {
@SerializedName("name")
private String name;
}

47
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java Normal file
View file

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public abstract class Cell {
@SerializedName("cell_type")
private String cellType;
@SerializedName("metadata")
private CellMetadata metadata;
@SerializedName("source")
private List<String> source;
public String getCellType() {
return cellType;
}
public CellMetadata getMetadata() {
return metadata;
}
public List<String> getSource() {
return source;
}
}

33
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CellMetadata.java Normal file
View file

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class CellMetadata {
@SerializedName("name")
private String name;
@SerializedName("tags")
private List<String> tags;
}

33
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CodeCell.java Normal file
View file

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class CodeCell extends Cell {
@SerializedName("outputs")
private List<Output> outputs;
public List<Output> getOutputs() {
return outputs;
}
}

33
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java Normal file
View file

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.Map;
/**
*
*/
public class DisplayData extends Output {
@SerializedName("data")
private Map<String, Object> data;
public Map<String, Object> getData() {
return data;
}
}

46
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java Normal file
View file

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Error extends Output {
@SerializedName("ename")
private String ename;
@SerializedName("evalue")
private String evalue;
@SerializedName("traceback")
private List<String> traceback;
public String getEname() {
return ename;
}
public String getEvalue() {
return evalue;
}
public List<String> getTraceback() {
return traceback;
}
}

36
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.Map;
/**
*
*/
public class ExecuteResult extends Output {
@SerializedName("execution_count")
private int executionCount;
@SerializedName("data")
private Map<String, Object> data;
public Map<String, Object> getData() {
return data;
}
}

32
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/HeadingCell.java Normal file
View file

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class HeadingCell extends Cell {
@SerializedName("level")
private int level;
public int getLevel() {
return level;
}
}

31
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Kernelspec.java Normal file
View file

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class Kernelspec {
@SerializedName("name")
private String name;
@SerializedName("display_name")
private String displayName;
}

41
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/LanguageInfo.java Normal file
View file

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class LanguageInfo {
@SerializedName("name")
private String name;
@SerializedName("codemirror_mode")
private Object codemirrorMode;
@SerializedName("file_extension")
private String fileExtension;
@SerializedName("mimetype")
private String mimetype;
@SerializedName("pygments_lexer")
private String pygmentsLexer;
}

24
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/MarkdownCell.java Normal file
View file

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
/**
*
*/
public class MarkdownCell extends Cell {
}

61
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Metadata.java Normal file
View file

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Metadata {
@SerializedName("kernelspec")
private Kernelspec kernelspec;
@SerializedName("language_info")
private LanguageInfo languageInfo;
@SerializedName("orig_nbformat")
private int origNbformat;
@SerializedName("title")
private String title;
@SerializedName("authors")
private List<Author> authors;
public Kernelspec getKernelspec() {
return kernelspec;
}
public LanguageInfo getLanguageInfo() {
return languageInfo;
}
public int getOrigNbformat() {
return origNbformat;
}
public String getTitle() {
return title;
}
public List<Author> getAuthors() {
return authors;
}
}

54
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Nbformat.java Normal file
View file

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Nbformat {
@SerializedName("metadata")
private Metadata metadata;
@SerializedName("nbformat")
private int nbformat;
@SerializedName("nbformat_minor")
private int nbformatMinor;
@SerializedName("cells")
private List<Cell> cells;
public Metadata getMetadata() {
return metadata;
}
public int getNbformat() {
return nbformat;
}
public int getNbformatMinor() {
return nbformatMinor;
}
public List<Cell> getCells() {
return cells;
}
}

28
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java Normal file
View file

@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public abstract class Output {
@SerializedName("output_type")
private String outputType;
}

24
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCell.java Normal file
View file

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
/**
*
*/
public class RawCell extends Cell {
}

28
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCellMetadata.java Normal file
View file

@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class RawCellMetadata extends CellMetadata {
@SerializedName("format")
private String format;
}

36
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Stream extends Output {
@SerializedName("name")
private String name;
@SerializedName("text")
private List<String> text;
public List<String> getText() {
return text;
}
}

48
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Note.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.zformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Note {
@SerializedName("name")
private String name;
@SerializedName("paragraphs")
private List<Paragraph> paragraphs;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<Paragraph> getParagraphs() {
return paragraphs;
}
public void setParagraphs(List<Paragraph> paragraphs) {
this.paragraphs = paragraphs;
}
}

70
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java Normal file
View file

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.zformat;
import com.google.gson.annotations.SerializedName;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
*
*/
public class Paragraph {
public static final String FINISHED = "FINISHED";
@SerializedName("text")
private String text;
@SerializedName("results")
private Result results; // It's a bit weird name
@SerializedName("id")
private String id;
@SerializedName("status")
private String status;
public Paragraph() {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-HHmmss");
this.id = dateFormat.format(new Date()) + "_" + super.hashCode();
this.status = FINISHED;
}
public String getText() {
return text;
}
public void setText(String text) {
this.text = text;
}
public Result getResults() {
return results;
}
public void setResults(Result results) {
this.results = results;
}
public String getId() {
return id;
}
public String getStatus() {
return status;
}
}

46
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java Normal file
View file

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.zformat;
import com.google.gson.annotations.SerializedName;
import java.util.List;
/**
*
*/
public class Result {
public static final String SUCCESS = "SUCCESS";
@SerializedName("code")
private String code;
@SerializedName("msg")
private List<TypeData> msg;
public Result(String code, List<TypeData> msg) {
this.code = code;
this.msg = msg;
}
public String getCode() {
return code;
}
public List<TypeData> getMsg() {
return msg;
}
}

47
zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/TypeData.java Normal file
View file

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.zformat;
import com.google.gson.annotations.SerializedName;
/**
*
*/
public class TypeData {
public static final String TABLE = "TABLE";
public static final String HTML = "HTML";
public static final String TEXT = "TEXT";
@SerializedName("type")
private String type;
@SerializedName("data")
private String data;
public TypeData(String type, String data) {
this.type = type;
this.data = data;
}
public String getType() {
return type;
}
public String getData() {
return data;
}
}

48
zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java Normal file
View file

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jupyter.nbformat;
import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.zeppelin.jupyter.JupyterUtil;
import org.apache.zeppelin.jupyter.zformat.Note;
import org.junit.Test;
/**
*
*/
public class JupyterUtilTest {
@Test
public void getNbFormat() {
InputStream resource = getClass().getResourceAsStream("/basic.ipynb");
Nbformat nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource));
assertTrue(nbformat.getCells().get(0) instanceof CodeCell);
resource = getClass().getResourceAsStream("/examples.ipynb");
nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource));
}
@Test
public void getNote() {
InputStream resource = getClass().getResourceAsStream("/examples.ipynb");
Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md");
}
}

149
zeppelin-jupyter/src/test/resources/basic.ipynb Normal file
View file

@ -0,0 +1,149 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# The first two lines of the A matrix represent the coordinates of each rotor in the X,Y plane,\n",
"# and the third line the direction in which they spin.\n",
"# Note that in this example the X axis is vertical, and the Y coordinates are in the top row of A.\n",
"A = np.array([[-0.17, 0.17, -0.25, 0.25, -0.33, 0.33],\n",
" [-0.35, -0.35, 0., 0., 0.35, 0.35],\n",
" [-0.1, 0.1, 0.1, -0.1, -0.1, 0.1 ]])"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Moore-Penrose pseudoinverse of A\n",
"B = np.linalg.pinv(A)\n",
"# normalize roll/pitch to the largest of both\n",
"# normalize yaw to 0.5\n",
"# and transpose\n",
"rp_max = B[:,0:1].max()\n",
"n = np.array([rp_max, rp_max, 2*B[:,2].max()])\n",
"B_nt = (B / n).T"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ -67 67 -256 256 -189 189]\n",
" [-197 -197 0 0 197 197]\n",
" [ -77 77 128 -128 -57 57]]\n"
]
}
],
"source": [
"# scale and round to 256 to return final coefficients\n",
"scale = 256\n",
"coeffs = np.around(scale * B_nt).astype(int)\n",
"print(coeffs)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<define name=\"ROLL_COEF\" value=\"{ -67, 67, -256, 256, -189, 189}\"/>\n",
"<define name=\"PITCH_COEF\" value=\"{-197, -197, 0, 0, 197, 197}\"/>\n",
"<define name=\"YAW_COEF\" value=\"{ -77, 77, 128, -128, -57, 57}\"/>\n"
]
}
],
"source": [
"# output defines\n",
"import string\n",
"rows = ['ROLL_COEF\" ', 'PITCH_COEF\" ', 'YAW_COEF\" ']\n",
"for i, r in enumerate(rows):\n",
" print('<define name=\"' + r + 'value=\"{' + string.join(['{:>4d}'.format(c) for c in coeffs[i]], ', ') + '}\"/>')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
""
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2.0
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

355
zeppelin-jupyter/src/test/resources/examples.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

113
zeppelin-server/pom.xml
View file

@ -34,11 +34,12 @@
<properties>
<!--library versions-->
<cxf.version>2.7.8</cxf.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
<jersey.version>2.22.2</jersey.version>
<hadoop-common.version>2.6.0</hadoop-common.version>
<quartz.scheduler.version>2.2.1</quartz.scheduler.version>
<jersey.servlet.version>1.13</jersey.servlet.version>
<javax.ws.rsapi.version>2.0-m10</javax.ws.rsapi.version>
<javax.ws.rsapi.version>2.0.1</javax.ws.rsapi.version>
<libpam4j.version>1.8</libpam4j.version>
<jna.version>4.1.0</jna.version>
@ -86,6 +87,12 @@
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-zengine</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@ -98,6 +105,45 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.version}</version>
<exclusions>
<exclusion>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
<version>${javax.ws.rsapi.version}</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
@ -131,54 +177,11 @@
<version>${jna.version}</version>
</dependency>
<!-- Needed for dependency conergence -->
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxrs</artifactId>
<version>${cxf.version}</version>
<exclusions>
<exclusion>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId>
<version>${cxf.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http-jetty</artifactId>
<version>${cxf.version}</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-api</artifactId>
<version>${cxf.version}</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.5.4</version>
</dependency>
<dependency>
@ -292,18 +295,6 @@
<version>${quartz.scheduler.version}</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-servlet</artifactId>
<version>${jersey.servlet.version}</version>
</dependency>
<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
<version>${javax.ws.rsapi.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>

10
zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java
View file

@ -297,6 +297,16 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
return userNameList;
}
public Map<String, String> getListRoles() {
Map<String, String> roles = new HashMap<>();
Iterator it = this.groupRolesMap.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
roles.put((String) pair.getValue(), "*");
}
return roles;
}
private Set<String> getRoleNamesForUser(String username, LdapContext ldapContext)
throws NamingException {
Set<String> roleNames = new LinkedHashSet<>();

48
zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java
View file

@ -109,6 +109,9 @@ import javax.naming.ldap.PagedResultsControl;
* <p># optional mapping from physical groups to logical application roles
* ldapRealm.rolesByGroup = \ LDN_USERS: user_role,\ NYK_USERS: user_role,\
* HKG_USERS: user_role,\ GLOBAL_ADMIN: admin_role,\ DEMOS: self-install_role
*
* <p># optional list of roles that are allowed to authenticate
* ldapRealm.allowedRolesForAuthentication = admin_role,user_role
*
* <p>ldapRealm.permissionsByRole=\ user_role = *:ToDoItemsJdo:*:*,\
* *:ToDoItem:*:*; \ self-install_role = *:ToDoItemsFixturesService:install:* ;
@ -176,6 +179,7 @@ public class LdapRealm extends JndiLdapRealm {
private String memberAttributeValueSuffix = "";
private final Map<String, String> rolesByGroup = new LinkedHashMap<String, String>();
private final List<String> allowedRolesForAuthentication = new ArrayList<String>();
private final Map<String, List<String>> permissionsByRole =
new LinkedHashMap<String, List<String>>();
@ -202,6 +206,29 @@ public class LdapRealm extends JndiLdapRealm {
}
}
/**
* This overrides the implementation of queryForAuthenticationInfo inside JndiLdapRealm.
* In addition to calling the super method for authentication it also tries to validate
* if this user has atleast one of the allowed roles for authentication. In case the property
* allowedRolesForAuthentication is empty this check always returns true.
*
* @param token the submitted authentication token that triggered the authentication attempt.
* @param ldapContextFactory factory used to retrieve LDAP connections.
* @return AuthenticationInfo instance representing the authenticated user's information.
* @throws NamingException if any LDAP errors occur.
*/
@Override
protected AuthenticationInfo queryForAuthenticationInfo(AuthenticationToken token,
LdapContextFactory ldapContextFactory)
throws NamingException {
AuthenticationInfo info = super.queryForAuthenticationInfo(token, ldapContextFactory);
// Credentials were verified. Verify that the principal has all allowedRulesForAuthentication
if (!hasAllowedAuthenticationRules(info.getPrincipals(), ldapContextFactory)) {
throw new NamingException("Principal does not have any of the allowedRolesForAuthentication");
}
return info;
}
/**
* Get groups from LDAP.
*
@ -231,6 +258,23 @@ public class LdapRealm extends JndiLdapRealm {
return simpleAuthorizationInfo;
}
private boolean hasAllowedAuthenticationRules(PrincipalCollection principals,
final LdapContextFactory ldapContextFactory)
throws NamingException {
boolean allowed = allowedRolesForAuthentication.isEmpty();
if (!allowed) {
Set<String> roles = getRoles(principals, ldapContextFactory);
for (String allowedRole: allowedRolesForAuthentication) {
if (roles.contains(allowedRole)) {
log.debug("Allowed role for user [" + allowedRole + "] found.");
allowed = true;
break;
}
}
}
return allowed;
}
private Set<String> getRoles(PrincipalCollection principals,
final LdapContextFactory ldapContextFactory)
throws NamingException {
@ -524,6 +568,10 @@ public class LdapRealm extends JndiLdapRealm {
this.memberAttributeValueSuffix = suffix;
}
public void setAllowedRolesForAuthentication(List<String> allowedRolesForAuthencation) {
this.allowedRolesForAuthentication.addAll(allowedRolesForAuthencation);
}
public void setRolesByGroup(Map<String, String> rolesByGroup) {
this.rolesByGroup.putAll(rolesByGroup);
}

4
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -588,17 +588,17 @@ public class NotebookRestApi {
throws IOException, IllegalArgumentException {
LOG.info("run note jobs {} ", noteId);
Note note = notebook.getNote(noteId);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note");
try {
note.runAll();
note.runAll(subject);
} catch (Exception ex) {
LOG.error("Exception from run", ex);
return new JsonResponse<>(Status.PRECONDITION_FAILED,
ex.getMessage() + "- Not selected or Invalid Interpreter bind").build();
}
return new JsonResponse<>(Status.OK).build();
}

27
zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java
View file

@ -17,12 +17,18 @@
package org.apache.zeppelin.rest;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.util.Util;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
/**
@ -52,4 +58,25 @@ public class ZeppelinRestApi {
public Response getVersion() {
return new JsonResponse<>(Response.Status.OK, "Zeppelin version", Util.getVersion()).build();
}
/**
* Set the log level for root logger
* @param request
* @param logLevel new log level for Rootlogger
* @return
*/
@PUT
@Path("log/level/{logLevel}")
public Response changeRootLogLevel(@Context HttpServletRequest request,
@PathParam("logLevel") String logLevel) {
Level level = Level.toLevel(logLevel);
if (logLevel.toLowerCase().equalsIgnoreCase(level.toString().toLowerCase())) {
Logger.getRootLogger().setLevel(level);
return new JsonResponse<>(Response.Status.OK).build();
} else {
return new JsonResponse<>(Response.Status.NOT_ACCEPTABLE,
"Please check LOG level specified. Valid values: DEBUG, ERROR, FATAL, "
+ "INFO, TRACE, WARN").build();
}
}
}

15
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -27,7 +27,6 @@ import javax.servlet.DispatcherType;
import javax.ws.rs.core.Application;
import org.apache.commons.lang.StringUtils;
import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet;
import org.apache.shiro.web.env.EnvironmentLoaderListener;
import org.apache.shiro.web.servlet.ShiroFilter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -286,7 +285,7 @@ public class ZeppelinServer extends Application {
final ServletHolder servletHolder = new ServletHolder(notebookWsServer);
servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);
final ServletContextHandler cxfContext = new ServletContextHandler(
final ServletContextHandler context = new ServletContextHandler(
ServletContextHandler.SESSIONS);
webapp.addServlet(servletHolder, "/ws/*");
@ -316,13 +315,15 @@ public class ZeppelinServer extends Application {
private static void setupRestApiContextHandler(WebAppContext webapp,
ZeppelinConfiguration conf) {
final ServletHolder cxfServletHolder = new ServletHolder(new CXFNonSpringJaxrsServlet());
cxfServletHolder.setInitParameter("javax.ws.rs.Application", ZeppelinServer.class.getName());
cxfServletHolder.setName("rest");
cxfServletHolder.setForcedPath("rest");
final ServletHolder servletHolder = new ServletHolder(
new org.glassfish.jersey.servlet.ServletContainer());
servletHolder.setInitParameter("javax.ws.rs.Application", ZeppelinServer.class.getName());
servletHolder.setName("rest");
servletHolder.setForcedPath("rest");
webapp.setSessionHandler(new SessionHandler());
webapp.addServlet(cxfServletHolder, "/api/*");
webapp.addServlet(servletHolder, "/api/*");
String shiroIniPath = conf.getShiroPath();
if (!StringUtils.isBlank(shiroIniPath)) {

29
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -19,15 +19,9 @@ package org.apache.zeppelin.socket;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.regex.Matcher;
@ -1696,6 +1690,23 @@ public class NotebookServer extends WebSocketServlet
p.setErrorMessage((String) fromMessage.get("errorMessage"));
p.setStatusWithoutNotification(status);
// Spell uses ISO 8601 formatted string generated from moment
String dateStarted = (String) fromMessage.get("dateStarted");
String dateFinished = (String) fromMessage.get("dateFinished");
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
try {
p.setDateStarted(df.parse(dateStarted));
} catch (ParseException e) {
LOG.error("Failed parse dateStarted", e);
}
try {
p.setDateFinished(df.parse(dateFinished));
} catch (ParseException e) {
LOG.error("Failed parse dateFinished", e);
}
addNewParagraphIfLastParagraphIsExecuted(note, p);
if (!persistNoteWithAuthInfo(conn, note, p)) {
return;

4
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -34,6 +34,7 @@ import org.apache.shiro.subject.Subject;
import org.apache.shiro.util.ThreadContext;
import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.realm.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.LdapRealm;
import org.mortbay.log.Log;
import org.slf4j.Logger;
@ -133,6 +134,9 @@ public class SecurityUtils {
} else if (name.equals("org.apache.zeppelin.realm.LdapRealm")) {
allRoles = ((LdapRealm) realm).getListRoles();
break;
} else if (name.equals("org.apache.zeppelin.realm.ActiveDirectoryGroupRealm")) {
allRoles = ((ActiveDirectoryGroupRealm) realm).getListRoles();
break;
}
}
if (allRoles != null) {

2
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -253,7 +253,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
}
}
@Test
@ -594,6 +593,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
CoreMatchers.equalTo("Howdy "));
Select dropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[1]"))));
dropDownMenu.selectByVisibleText("Alice");
collector.checkThat("After selection in drop down menu, output should display the newly selected option",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(),

14
zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
View file

@ -32,6 +32,8 @@ import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
public class SparkParagraphIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class);
@ -182,10 +184,16 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
);
}
WebElement paragraph1Result = driver.findElement(By.xpath(
getParagraphXPath(1) + "//div[contains(@id,\"_graph\")]/div/div/div/div/div[1]"));
// Age, Job, Marital, Education, Balance
List<WebElement> tableHeaders = driver.findElements(By.cssSelector("span.ui-grid-header-cell-label"));
String headerNames = "";
for(WebElement header : tableHeaders) {
headerNames += header.getText().toString() + "|";
}
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph1Result.getText().toString(), CoreMatchers.equalTo("age\n▼\njob\n▼\nmarital\n▼\neducation\n▼\nbalance\n▼\n30 unemployed married primary 1787"));
headerNames, CoreMatchers.equalTo("Age|Job|Marital|Education|Balance|"));
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testSqlSpark", e);
}

10
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -429,13 +429,19 @@ public abstract class AbstractTestRestApi {
LOG.info("{} - {}", postMethod.getStatusCode(), postMethod.getStatusText());
Pattern pattern = Pattern.compile("JSESSIONID=([a-zA-Z0-9-]*)");
Header[] setCookieHeaders = postMethod.getResponseHeaders("Set-Cookie");
String jsessionId = null;
for (Header setCookie : setCookieHeaders) {
java.util.regex.Matcher matcher = pattern.matcher(setCookie.toString());
if (matcher.find()) {
return matcher.group(1);
jsessionId = matcher.group(1);
}
}
return StringUtils.EMPTY;
if (jsessionId != null) {
return jsessionId;
} else {
return StringUtils.EMPTY;
}
}
protected static boolean userAndPasswordAreNotBlank(String user, String pwd) {

2
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
View file

@ -167,7 +167,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("paragraph col width check failed", 9.0, p.getConfig().get("colWidth"));
assertTrue("paragraph show title check failed", ((boolean) p.getConfig().get("title")));
Map graph = ((List<Map>)p.getConfig().get("results")).get(0);
String mode = graph.get("mode").toString();
String mode = ((Map)graph.get("graph")).get("mode").toString();
assertEquals("paragraph graph mode check failed", "pieChart", mode);
}
}

3
zeppelin-web/.eslintrc
View file

@ -58,6 +58,7 @@
"no-undef": 2,
"no-unused-vars": [2, { "vars": "local", "args": "none" }],
"strict": [2, "global"],
"max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}]
"max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}],
"linebreak-style": 0
}
}

2
zeppelin-web/bower.json
View file

@ -30,8 +30,6 @@
"ngtoast": "~2.0.0",
"ng-focus-if": "~1.0.2",
"bootstrap3-dialog": "bootstrap-dialog#~1.34.7",
"handsontable": "~0.24.2",
"moment-duration-format": "^1.3.0",
"select2": "^4.0.3",
"MathJax": "2.7.0",
"ngclipboard": "^1.1.1"

5
zeppelin-web/karma.conf.js
View file

@ -81,11 +81,6 @@ module.exports = function(config) {
'bower_components/ngtoast/dist/ngToast.js',
'bower_components/ng-focus-if/focusIf.js',
'bower_components/bootstrap3-dialog/dist/js/bootstrap-dialog.min.js',
'bower_components/zeroclipboard/dist/ZeroClipboard.js',
'bower_components/moment/moment.js',
'bower_components/pikaday/pikaday.js',
'bower_components/handsontable/dist/handsontable.js',
'bower_components/moment-duration-format/lib/moment-duration-format.js',
'bower_components/select2/dist/js/select2.js',
'bower_components/MathJax/MathJax.js',
'bower_components/clipboard/dist/clipboard.js',

13
zeppelin-web/package.json
View file

@ -9,8 +9,8 @@
"clean": "rimraf dist && rimraf .tmp",
"postinstall": "bower install --silent",
"prebuild": "npm-run-all clean lint:once",
"build:dist": "grunt pre-webpack-dist && webpack && grunt post-webpack-dist",
"build:ci": "grunt pre-webpack-ci && webpack && grunt post-webpack-dist",
"build:dist": "npm-run-all prebuild && grunt pre-webpack-dist && webpack && grunt post-webpack-dist",
"build:ci": "npm-run-all prebuild && grunt pre-webpack-ci && webpack && grunt post-webpack-dist",
"lint:watch": "esw --watch src",
"lint:once": "eslint src",
"predev": "grunt pre-webpack-dev",
@ -18,14 +18,18 @@
"dev:helium": "HELIUM_BUNDLE_DEV=true webpack-dev-server --hot",
"dev:watch": "grunt watch-webpack-dev",
"dev": "npm-run-all --parallel dev:server lint:watch dev:watch",
"pretest": "npm install karma-phantomjs-launcher babel-polyfill",
"test": "karma start karma.conf.js"
},
"dependencies": {
"angular-viewport-watch": "github:shahata/angular-viewport-watch",
"github-markdown-css": "2.6.0",
"angular-ui-grid": "^4.0.4",
"grunt-angular-templates": "^0.5.7",
"grunt-dom-munger": "^3.4.0",
"headroom.js": "^0.9.3"
"headroom.js": "^0.9.3",
"moment": "^2.18.1",
"moment-duration-format": "^1.3.0",
"scrollmonitor": "^1.2.3"
},
"devDependencies": {
"autoprefixer": "^6.5.4",
@ -73,6 +77,7 @@
"karma-coverage": "^1.1.1",
"karma-jasmine": "~1.0.2",
"karma-sourcemap-loader": "^0.3.7",
"karma-phantomjs-launcher": "^1.0.4",
"karma-webpack": "^1.8.1",
"load-grunt-tasks": "^0.4.0",
"ng-annotate-loader": "^0.2.0",

11
zeppelin-web/src/WEB-INF/web.xml
View file

@ -23,11 +23,12 @@
<display-name>zeppelin-web</display-name>
<servlet>
<servlet-name>default</servlet-name>
<servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>com.sun.jersey.config.property.packages</param-name>
<param-value>org.apache.zeppelin.rest</param-value>
</init-param>
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>jersey.config.server.provider.packages</param-name>
<param-value>org.apache.zeppelin.rest</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>

17
zeppelin-web/src/app/app.js
View file

@ -18,6 +18,12 @@
import 'headroom.js'
import 'headroom.js/dist/angular.headroom'
import 'scrollmonitor/scrollMonitor.js'
import 'angular-viewport-watch/angular-viewport-watch.js'
import 'angular-ui-grid/ui-grid.css'
import 'angular-ui-grid'
const requiredModules = [
'ngCookies',
'ngAnimate',
@ -37,6 +43,17 @@ const requiredModules = [
'focus-if',
'ngResource',
'ngclipboard',
'angularViewportWatch',
'ui.grid',
'ui.grid.exporter',
'ui.grid.edit', 'ui.grid.rowEdit',
'ui.grid.selection',
'ui.grid.cellNav', 'ui.grid.pinning',
'ui.grid.grouping',
'ui.grid.emptyBaseLayer',
'ui.grid.resizeColumns', 'ui.grid.moveColumns',
'ui.grid.pagination',
'ui.grid.saveState',
]
// headroom should not be used for CI, since we have to execute some integration tests.

2
zeppelin-web/src/app/configuration/configuration.controller.js
View file

@ -34,7 +34,7 @@ function ConfigurationCtrl ($scope, $rootScope, $http, baseUrlSrv, ngToast) {
timeout: '3000'
})
setTimeout(function () {
window.location.replace('/')
window.location = baseUrlSrv.getBase()
}, 3000)
}
console.log('Error %o %o', status, data.message)

2
zeppelin-web/src/app/credential/credential.controller.js
View file

@ -40,7 +40,7 @@ function CredentialCtrl ($scope, $rootScope, $http, baseUrlSrv, ngToast) {
timeout: '3000'
})
setTimeout(function () {
window.location.replace('/')
window.location = baseUrlSrv.getBase()
}, 3000)
}
console.log('Error %o %o', status, data.message)

201
zeppelin-web/src/app/handsontable/handsonHelper.js
View file

@ -1,201 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* HandsonHelper class
*/
export default class HandsonHelper {
constructor (columns, rows, comment) {
this.columns = columns || []
this.rows = rows || []
this.comment = comment || ''
this._numericValidator = this._numericValidator.bind(this)
}
getHandsonTableConfig (columns, columnNames, resultRows) {
let self = this
return {
colHeaders: columnNames,
data: resultRows,
rowHeaders: false,
stretchH: 'all',
sortIndicator: true,
columns: columns,
columnSorting: true,
contextMenu: false,
manualColumnResize: true,
manualRowResize: true,
readOnly: true,
readOnlyCellClassName: '',
fillHandle: false,
fragmentSelection: true,
disableVisualSelection: true,
cells: function (ro, co, pro) {
let cellProperties = {}
let colType = columns[co].type
cellProperties.renderer = function (instance, td, row, col, prop, value, cellProperties) {
self._cellRenderer(instance, td, row, col, prop, value, cellProperties, colType)
}
return cellProperties
},
afterGetColHeader: function (col, TH) {
let instance = this
let menu = self._buildDropDownMenu(columns[col].type)
let button = self._buildTypeSwitchButton()
self._addButtonMenuEvent(button, menu)
Handsontable.Dom.addEvent(menu, 'click', function (event) {
if (event.target.nodeName === 'LI') {
self._setColumnType(columns, event.target.data.colType, instance, col)
}
})
if (TH.firstChild.lastChild.nodeName === 'BUTTON') {
TH.firstChild.removeChild(TH.firstChild.lastChild)
}
TH.firstChild.appendChild(button)
TH.style['white-space'] = 'normal'
}
}
}
/*
** Private Service Functions
*/
_addButtonMenuEvent (button, menu) {
Handsontable.Dom.addEvent(button, 'click', function (event) {
let changeTypeMenu
let position
let removeMenu
document.body.appendChild(menu)
event.preventDefault()
event.stopImmediatePropagation()
changeTypeMenu = document.querySelectorAll('.changeTypeMenu')
for (let i = 0, len = changeTypeMenu.length; i < len; i++) {
changeTypeMenu[i].style.display = 'none'
}
menu.style.display = 'block'
position = button.getBoundingClientRect()
menu.style.top = (position.top + (window.scrollY || window.pageYOffset)) + 2 + 'px'
menu.style.left = (position.left) + 'px'
removeMenu = function (event) {
if (menu.parentNode) {
menu.parentNode.removeChild(menu)
}
}
Handsontable.Dom.removeEvent(document, 'click', removeMenu)
Handsontable.Dom.addEvent(document, 'click', removeMenu)
})
}
_buildDropDownMenu (activeCellType) {
let menu = document.createElement('UL')
let types = ['text', 'numeric', 'date']
let item
menu.className = 'changeTypeMenu'
for (let i = 0, len = types.length; i < len; i++) {
item = document.createElement('LI')
if ('innerText' in item) {
item.innerText = types[i]
} else {
item.textContent = types[i]
}
item.data = {'colType': types[i]}
if (activeCellType === types[i]) {
item.className = 'active'
}
menu.appendChild(item)
}
return menu
}
_buildTypeSwitchButton () {
let button = document.createElement('BUTTON')
button.innerHTML = '\u25BC'
button.className = 'changeType'
return button
}
_isNumeric (value) {
if (!isNaN(value)) {
if (value.length !== 0) {
if (Number(value) <= Number.MAX_SAFE_INTEGER && Number(value) >= Number.MIN_SAFE_INTEGER) {
return true
}
}
}
return false
}
_cellRenderer (instance, td, row, col, prop, value, cellProperties, colType) {
if (colType === 'numeric' && this._isNumeric(value)) {
cellProperties.format = '0,0.[00000]'
td.style.textAlign = 'left'
// eslint-disable-next-line prefer-rest-params
Handsontable.renderers.NumericRenderer.apply(this, arguments)
} else if (value.length > '%html'.length && value.substring(0, '%html '.length) === '%html ') {
td.innerHTML = value.substring('%html'.length)
} else {
// eslint-disable-next-line prefer-rest-params
Handsontable.renderers.TextRenderer.apply(this, arguments)
}
}
_dateValidator (value, callback) {
let d = moment(value)
return callback(d.isValid())
}
_numericValidator (value, callback) {
return callback(this._isNumeric(value))
}
_setColumnType (columns, type, instance, col) {
columns[col].type = type
this._setColumnValidator(columns, col)
instance.updateSettings({columns: columns})
instance.validateCells(null)
if (this._isColumnSorted(instance, col)) {
instance.sort(col, instance.sortOrder)
}
}
_isColumnSorted (instance, col) {
return instance.sortingEnabled && instance.sortColumn === col
}
_setColumnValidator (columns, col) {
if (columns[col].type === 'numeric') {
columns[col].validator = this._numericValidator
} else if (columns[col].type === 'date') {
columns[col].validator = this._dateValidator
} else {
columns[col].validator = null
}
}
}

166
zeppelin-web/src/app/home/home.css
View file

@ -32,14 +32,14 @@ body {
background: white;
}
.displayNavBar {
display: inline !important;
}
body.asIframe {
padding-top: 0;
}
.displayNavBar {
display: inline !important;
}
body .navbar {
margin-bottom: 0;
}
@ -59,58 +59,6 @@ body .navbar {
text-decoration: none;
}
.navbar-inverse {
background: #3071a9;
color: #fff;
border-color: #3071a9;
font-size: 18px;
}
.navbar-inverse .navbar-nav > .open > a,
.navbar-inverse .navbar-nav > .open > a:hover,
.navbar-inverse .navbar-nav > .open > a:focus {
color: #fff;
background: #3071a9;
}
.navbar-inverse .navbar-toggle:hover,
.navbar-inverse .navbar-toggle:focus {
background: #3071a9;
}
.navbar-inverse .navbar-nav > li > a:hover,
.navbar-inverse .navbar-nav > li > a:focus {
color: #fff;
background: rgba(0, 0, 0, 0.2);
}
.navbar-inverse .navbar-toggle {
border-color: #FFFFFF;
}
.navbar-inverse .navbar-nav > .active > a,
.navbar-inverse .navbar-nav > .active > a:hover,
.navbar-inverse .navbar-nav > .active > a:focus {
color: #fff;
background: #080808;
}
.navbar-inverse .navbar-nav > li > a {
color: #FFFFFF;
}
.navbar-inverse .navbar-brand {
color: #fff;
text-decoration: none;
font-family: 'Patua One', cursive;
font-size: 32px;
}
a.navbar-brand:hover {
color: #fff !important;
}
/* Css for the Notebook Dropdown */
.expandable ul {
@ -248,45 +196,6 @@ a.navbar-brand:hover {
border-radius:6px 0 6px 6px;
}
@media (max-width: 767px) {
.navbar-inverse .navbar-nav .open .dropdown-menu > li > a,
#notebook-list li a {
color: #D3D3D3;
}
.navbar-nav .open .dropdown-menu > .scrollbar-container > li > a,
#notebook-list li a {
padding: 5px 15px 5px 25px;
line-height: 20px;
}
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a {
color: #D3D3D3;
}
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:hover,
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:focus,
#notebook-list li a:hover {
color: #fff;
background: transparent;
}
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a,
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:hover,
.navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:focus {
color: #fff;
background: #080808;
}
.navbar-inverse .navbar-nav .open .dropdown-menu .divider {
background: #3071A9;
}
.navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form {
border-color: #3071A9;
}
}
#main {
padding: 10px;
height: 100%;
@ -335,14 +244,6 @@ a.navbar-brand:hover {
border-color: #357ebd;
}
.username {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
max-width: 120px;
display: inline-block;
}
.server-connected {
padding-top: 12px;
color: #00CC00;
@ -455,40 +356,6 @@ This part should be removed when new version of bootstrap handles this issue.
z-index: 10003 !important;
}
#noteImportModal .modal-body {
min-height: 420px;
overflow: hidden;
}
#noteImportModal .modal-footer {
min-height: 65px;
}
#noteImportModal .display-inline a {
background: #fff;
border: 1px solid #ddd;
border-radius: 5px;
color: #999;
cursor: pointer;
display: block;
float: left;
font-size: 98px;
height: 240px;
margin: 0 10px 16px;
padding-top: 60px;
text-align: center;
text-decoration: none;
width: 264px;
}
#noteImportModal .display-inline a:hover {
background: #eee;
}
#noteImportModal .display-inline a p {
font-size: 14px;
}
/* ------------------------------------------- */
/* Slide Top
/* ------------------------------------------- */
@ -824,28 +691,3 @@ This part should be removed when new version of bootstrap handles this issue.
.bootstrap-dialog.type-primary .modal-header {
background: #3071a9;
}
/* About Zeppelin */
.about {
height: 200px;
padding: 25px;
}
.about .logo {
padding-top: 30px;
}
.about .logo img {
width: 95%;
}
.about .content {
text-align: center;
}
.about .content h3 {
font-family: 'Patua One';
color: #3071A9;
font-size: 30px;
margin: 0 auto;
}

2
zeppelin-web/src/app/home/notebook-template.html
View file

@ -56,7 +56,7 @@ limitations under the License.
<div ng-mouseenter="showFolderButton=true"
ng-mouseleave="showFolderButton=false">
<a style="text-decoration: none; cursor: pointer;" ng-click="toggleFolderNode(node)">
<i style="font-size: 10px;" ng-class="node.hidden ? 'icon-folder' : 'icon-folder-alt'"></i> {{getNoteName(node)}}
<i style="font-size: 10px;" ng-class="node.hidden ? 'fa fa-folder' : 'fa fa-folder-open'"></i> {{getNoteName(node)}}
</a>
<a ng-if="!node.isTrash" href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;"
ng-controller="NotenameCtrl as notenamectrl" ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}">

2
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -101,7 +101,7 @@ function InterpreterCtrl ($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeo
timeout: '3000'
})
setTimeout(function () {
window.location.replace('/')
window.location = baseUrlSrv.getBase()
}, 3000)
}
console.log('Error %o %o', status, data.message)

2
zeppelin-web/src/app/jobmanager/jobs/job.controller.js
View file

@ -12,6 +12,8 @@
* limitations under the License.
*/
import moment from 'moment'
import { ParagraphStatus, } from '../../notebook/paragraph/paragraph.status'
angular.module('zeppelinWebApp').controller('JobCtrl', JobCtrl)

2
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -156,7 +156,7 @@ limitations under the License.
<strong>{{revision.message}}</strong>
</span>
<span class="revisionDate">
<em>{{moment.unix(revision.time).format('MMMM Do YYYY, h:mm a')}}</em>
<em>{{formatRevisionDate(revision.time)}}</em>
</span>
</a>
</li>

Some files were not shown because too many files have changed in this diff Show more