Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-732-up

Browse source
This commit is contained in:
Lee moon soo 2016-06-20 23:26:29 -07:00
commit d43ab574bb
77 changed files with 2176 additions and 1334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored
View file

@ -32,7 +32,7 @@ conf/credentials.json
spark/dependency-reduced-pom.xml
reports
#webapp
# webapp
zeppelin-web/node_modules
zeppelin-web/dist
zeppelin-web/.tmp
@ -41,8 +41,9 @@ zeppelin-web/bower_components
**nbproject/
**node/
#R
# R
/r/lib/
.Rhistory
# project level
/logs/
@ -80,7 +81,7 @@ Thumbs.db
.project
.settings/
#intelliJ IDEA project files
# intelliJ IDEA project files
.idea/
*.iml

3
.travis.yml
View file

@ -67,8 +67,7 @@ before_install:
- echo 'R_LIBS=~/R' > ~/.Renviron
- R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')"
- export R_LIBS='~/R'
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1600x1024x16"
install:
- mvn $BUILD_FLAG $PROFILE -B

31
conf/shiro.ini
View file

@ -25,19 +25,44 @@ user3 = password4, role2
# Sample LDAP configuration, for user Authentication, currently tested for single Realm
[main]
#ldapRealm = org.apache.shiro.realm.ldap.JndiLdapRealm
#ldapRealm.userDnTemplate = cn={0},cn=engg,ou=testdomain,dc=testdomain,dc=com
#ldapRealm.contextFactory.url = ldap://ldaphost:389
### A sample for configuring Active Directory Realm
#activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
#activeDirectoryRealm.systemUsername = userNameA
#activeDirectoryRealm.systemPassword = passwordA
#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
#activeDirectoryRealm.url = ldap://ldap.test.com:389
#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
#activeDirectoryRealm.authorizationCachingEnabled = false
### A sample for configuring LDAP Directory Realm
#ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
## search base for ldap groups (only relevant for LdapGroupRealm):
#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
#ldapRealm.contextFactory.url = ldap://ldap.test.com:389
#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
### If caching of user is required then uncomment below lines
#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
#securityManager.cacheManager = $cacheManager
securityManager.sessionManager = $sessionManager
# 86,400,000 milliseconds = 24 hour
securityManager.sessionManager.globalSessionTimeout = 86400000
shiro.loginUrl = /api/login
[roles]
role1 = *
role2 = *
role3 = *
[urls]
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# authc means Form based Auth Security
# To enfore security, comment the line below and uncomment the next one
/api/version = anon
/** = anon

2
conf/zeppelin-site.xml.template
View file

@ -178,7 +178,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

2
docs/assets/themes/zeppelin/css/style.css
View file

@ -433,6 +433,8 @@ a.anchor {
word-break: keep-all;
-webkit-overflow-scrolling: touch;
font-size: 90%;
margin-top: 16px;
margin-bottom: 16px;
}
.content table th {
font-weight: bold;

6
docs/index.md
View file

@ -124,9 +124,9 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
####Quick Start
* Getting Started
* [Quick Start](./install/install.html) for basic instructions on installing Zeppelin
* [Configuration](./install/install.html#zeppelin-configuration) lists for Zeppelin
* [Explore Apache Zeppelin UI](./quickstart/explorezeppelinui.html): basic components of Zeppelin home
* [Quick Start](./install/install.html) for basic instructions on installing Apache Zeppelin
* [Configuration](./install/install.html#apache-zeppelin-configuration) lists for Apache Zeppelin
* [Explore Apache Zeppelin UI](./quickstart/explorezeppelinui.html): basic components of Apache Zeppelin home
* [Tutorial](./quickstart/tutorial.html): a short walk-through tutorial that uses Apache Spark backend
* Basic Feature Guide
* [Dynamic Form](./manual/dynamicform.html): a step by step guide for creating dynamic forms

159
docs/install/install.md
View file

@ -19,38 +19,113 @@ limitations under the License.
-->
{% include JB/setup %}
## Zeppelin Installation
Welcome to your first trial to explore Zeppelin!
# Quick Start
Welcome to your first trial to explore Apache Zeppelin!
This page will help you to get started and here is the list of topics covered.
In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below.
* [Installation](#installation)
* [Downloading Binary Package](#downloading-binary-package)
* [Building from Source](#building-from-source)
* [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line)
* [Start Zeppelin](#start-zeppelin)
* [Stop Zeppelin](#stop-zeppelin)
* [(Optional) Start Apache Zeppelin with a service manager](#optional-start-apache-zeppelin-with-a-service-manager)
* [What is the next?](#what-is-the-next)
* [Apache Zeppelin Configuration](#apache-zeppelin-configuration)
### Install with Binary Package
## Installation
If you want to install Zeppelin with latest binary package, please visit [this page](http://zeppelin.apache.org/download.html).
Apache Zeppelin officially supports and is tested on next environments.
### Build from Zeppelin Source
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>Oracle JDK</td>
<td>1.7 <br /> (set <code>JAVA_HOME</code>)</td>
</tr>
<tr>
<td>OS</td>
<td>Mac OSX <br /> Ubuntu 14.X <br /> CentOS 6.X <br /> Windows 7 Pro SP1</td>
</tr>
</table>
You can also build Zeppelin from the source.
There are two options to install Apache Zeppelin on your machine. One is [downloading prebuild binary package](#downloading-binary-package) from the archive.
You can download not only the latest stable version but also the older one if you need.
The other option is [building from the source](#building-from-source).
Although it can be unstable somehow since it is on development status, you can explore newly added feature and change it as you want.
#### Prerequisites for build
* Java 1.7
* Git
* Maven(3.1.x or higher)
* Node.js Package Manager
### Downloading Binary Package
If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/zeppelin/blob/master/README.md) for the details.
If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html).
After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section.
### Building from Source
If you want to build from the source, the software below needs to be installed on your system.
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>Git</td>
<td></td>
</tr>
<tr>
<td>Maven</td>
<td>3.1.x or higher</td>
</tr>
</table>
Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation.
[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example.
If you don't have it installed yet, please check [Before Build](https://github.com/apache/zeppelin/blob/master/README.md#before-build) section and follow step by step instructions from there.
## Zeppelin Start / Stop
####1. Clone Apache Zeppelin repository
```
git clone https://github.com/apache/zeppelin.git
```
####2. Build source with options
Each interpreters requires different build options. For the further information about options, please see [Build](https://github.com/apache/zeppelin#build) section.
```
mvn clean package -DskipTests [Options]
```
Here are some examples with several options
```
# basic build
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
# spark-cassandra integration
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests
# with CDH
mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests
# with MapR
mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests
```
For the further information about building with source, please see [README.md](https://github.com/apache/zeppelin/blob/master/README.md) in Zeppelin repository.
## Starting Apache Zeppelin with Command Line
#### Start Zeppelin
```
bin/zeppelin-daemon.sh start
```
If you are using Windows
```
bin\zeppelin.cmd
```
After successful start, visit [http://localhost:8080](http://localhost:8080) with your web browser.
#### Stop Zeppelin
@ -59,21 +134,28 @@ After successful start, visit [http://localhost:8080](http://localhost:8080) wit
bin/zeppelin-daemon.sh stop
```
#### Start Zeppelin with a service manager such as upstart
#### (Optional) Start Apache Zeppelin with a service manager
Zeppelin can auto start as a service with an init script, such as services managed by upstart.
> **Note :** The below description was written based on Ubuntu Linux.
The following is an example upstart script to be saved as `/etc/init/zeppelin.conf`
This example has been tested with Ubuntu Linux.
Apache Zeppelin can be auto started as a service with an init script, such as services managed by **upstart**.
The following is an example of upstart script to be saved as `/etc/init/zeppelin.conf`
This also allows the service to be managed with commands such as
`sudo service zeppelin start`
`sudo service zeppelin stop`
`sudo service zeppelin restart`
```
sudo service zeppelin start
sudo service zeppelin stop
sudo service zeppelin restart
```
Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart`
Other service managers could use a similar approach with the `upstart` argument passed to the `zeppelin-daemon.sh` script.
##### zeppelin.conf
```
bin/zeppelin-daemon.sh upstart
```
**zeppelin.conf**
```
description "zeppelin"
@ -93,15 +175,16 @@ chdir /usr/share/zeppelin
exec bin/zeppelin-daemon.sh upstart
```
#### Running on Windows
## What is the next?
Congratulation on your successful Apache Zeppelin installation! Here are two next steps you might need.
```
bin\zeppelin.cmd
```
* For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html)
* After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend
* If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration)
## Zeppelin Configuration
## Apache Zeppelin Configuration
You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
You can configure Apache Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority.
<table class="table-configuration">
<tr>
@ -210,13 +293,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN</td>
<td>zeppelin.notebook.homescreen</td>
<td></td>
<td>A notebook id displayed in Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
<td>A notebook id displayed in Apache Zeppelin homescreen <br />i.e. 2A94M5J1Z</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE</td>
<td>zeppelin.notebook.homescreen.hide</td>
<td>false</td>
<td>This value can be "true" when to hide the notebook id set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
<td>This value can be "true" when to hide the notebook id set by <code>ZEPPELIN_NOTEBOOK_HOMESCREEN</code> on the Apache Zeppelin homescreen. <br />For the further information, please read <a href="../manual/notebookashomepage.html">Customize your Zeppelin homepage</a>.</td>
</tr>
<tr>
<td>ZEPPELIN_WAR_TEMPDIR</td>
@ -228,13 +311,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_DIR</td>
<td>zeppelin.notebook.dir</td>
<td>notebook</td>
<td>The root directory where Zeppelin notebook directories are saved</td>
<td>The root directory where notebook directories are saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_BUCKET</td>
<td>zeppelin.notebook.s3.bucket</td>
<td>zeppelin</td>
<td>S3 Bucket where Zeppelin notebook files will be saved</td>
<td>S3 Bucket where notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_S3_USER</td>
@ -270,7 +353,7 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>ZEPPELIN_NOTEBOOK_AZURE_SHARE</td>
<td>zeppelin.notebook.azure.share</td>
<td>zeppelin</td>
<td>Share where the Zeppelin notebook files will be saved</td>
<td>Share where the notebook files will be saved</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_AZURE_USER</td>
@ -291,13 +374,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>org.apache.zeppelin.spark.SparkInterpreter,<br />org.apache.zeppelin.spark.PySparkInterpreter,<br />org.apache.zeppelin.spark.SparkSqlInterpreter,<br />org.apache.zeppelin.spark.DepInterpreter,<br />org.apache.zeppelin.markdown.Markdown,<br />org.apache.zeppelin.shell.ShellInterpreter,<br />
...
</td>
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in Zeppelin notebook paragraph. </td>
<td>Comma separated interpreter configurations [Class] <br /> The first interpreter will be a default value. <br /> It means only the first interpreter in this list can be available without <code>%interpreter_name</code> annotation in notebook paragraph. </td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETER_DIR</td>
<td>zeppelin.interpreter.dir</td>
<td>interpreter</td>
<td>Zeppelin interpreter directory</td>
<td>Interpreter directory</td>
</tr>
<tr>
<td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td>

36
docs/interpreter/jdbc.md
View file

@ -18,6 +18,7 @@ This interpreter lets you create a JDBC connection to any data source, by now it
* Apache Hive
* Apache Drill
* Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver)
* Apache Phoenix
* Apache Tajo
If someone else used another database please report how it works to improve functionality.
@ -236,6 +237,41 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
<td></td>
</tr>
</table>
#### Phoenix
##### Properties
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Value</th>
</tr>
<tr>
<td>phoenix.driver</td>
<td>org.apache.phoenix.jdbc.PhoenixDriver</td>
</tr>
<tr>
<td>phoenix.url</td>
<td>jdbc:phoenix:localhost:2181:/hbase-unsecure</td>
</tr>
<tr>
<td>phoenix.user</td>
<td>phoenix_user</td>
</tr>
<tr>
<td>phoenix.password</td>
<td>phoenix_password</td>
</tr>
</table>
##### Dependencies
<table class="table-configuration">
<tr>
<th>Artifact</th>
<th>Excludes</th>
</tr>
<tr>
<td>org.apache.phoenix:phoenix-core:4.4.0-HBase-1.0</td>
<td></td>
</tr>
</table>
#### Tajo
##### Properties
<table class="table-configuration">

30
docs/interpreter/python.md
View file

@ -68,8 +68,8 @@ print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
* Code-completion is currently not implemented.
## Matplotlib integration
The python interpreter can display matplotlib graph with the function **_zeppelin_show()_**
You need to already have matplotlib module installed and a running XServer to use this functionality !
The python interpreter can display matplotlib graph with the function `zeppelin_show()`.
You need to have matplotlib module installed and a XServer running to use this functionality !
```python
%python
@ -90,28 +90,6 @@ zeppelin_show(plt,height='150px')
[![pythonmatplotlib](../interpreter/screenshots/pythonMatplotlib.png)](/docs/interpreter/screenshots/pythonMatplotlib.png)
## Technical description - Interpreter architecture
## Technical description
### Dev prerequisites
* Python 2 and 3 installed with py4j (0.9.2) and matplotlib (1.31 or later) installed on each
* Tests only checks the interpreter logic and starts any Python process ! Python process is mocked with a class that simply output it input.
* Make sure the code wrote in bootstrap.py and bootstrap_input.py is Python2 and 3 compliant.
* Use PEP8 convention for python code.
### Technical overview
* When interpreter is starting it launches a python process inside a Java ProcessBuilder. Python is started with -i (interactive mode) and -u (unbuffered stdin, stdout and stderr) options. Thus the interpreter has a "sleeping" python process.
* Interpreter sends command to python with a Java `outputStreamWiter` and read from an `InputStreamReader`. To know when stop reading stdout, interpreter sends `print "*!?flush reader!?*"`after each command and reads stdout until he receives back the `*!?flush reader!?*`.
* When interpreter is starting, it sends some Python code (bootstrap.py and bootstrap_input.py) to initialize default behavior and functions (`help(), z.input()...`). bootstrap_input.py is sent only if py4j library is detected inside Python process.
* [Py4J](https://www.py4j.org/) python and java libraries is used to load Input zeppelin Java class into the python process (make java code with python code !). Therefore the interpreter can directly create Zeppelin input form inside the Python process (and eventually with some python variable already defined). JVM opens a random open port to be accessible from python process.
* JavaBuilder can't send SIGINT signal to interrupt paragraph execution. Therefore interpreter directly send a `kill SIGINT PID` to python process to interrupt execution. Python process catch SIGINT signal with some code defined in bootstrap.py
* Matplotlib display feature is made with SVG export (in string) and then displays it with html code.
For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md)

2
docs/rest-api/rest-configuration.md
View file

@ -73,7 +73,7 @@ limitations under the License.
"zeppelin.server.context.path": "/",
"zeppelin.ssl.keystore.type": "JKS",
"zeppelin.ssl.truststore.path": "truststore",
"zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter",
"zeppelin.interpreters": "org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter",
"zeppelin.ssl": "false",
"zeppelin.notebook.autoInterpreterBinding": "true",
"zeppelin.notebook.homescreen": "",

32
docs/security/shiroauthentication.md
View file

@ -69,4 +69,36 @@ user2 = password3
Those combinations are defined in the `conf/shiro.ini` file.
####5. Groups and permissions (optional)
In case you want to leverage user groups and permissions, use one of the following configuration for LDAP or AD under `[main]` segment of shiro.ini
```
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
activeDirectoryRealm.systemUsername = userNameA
activeDirectoryRealm.systemPassword = passwordA
activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
activeDirectoryRealm.url = ldap://ldap.test.com:389
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
activeDirectoryRealm.authorizationCachingEnabled = false
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
# search base for ldap groups (only relevant for LdapGroupRealm):
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
ldapRealm.contextFactory.url = ldap://ldap.test.com:389
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
```
also define roles/groups that you want to have in system, like below;
```
[roles]
admin = *
hr = *
finance = *
group1 = *
```
All of above configurations are defined in the `conf/shiro.ini` file.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).

21
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -99,6 +99,9 @@ public class JDBCInterpreter extends Interpreter {
static final String EMPTY_COLUMN_VALUE = "";
private final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use";
private final String CONCURRENT_EXECUTION_COUNT = "zeppelin.jdbc.concurrent.max_connection";
private final HashMap<String, Properties> propertiesMap;
private final Map<String, Statement> paragraphIdStatementMap;
@ -434,8 +437,10 @@ public class JDBCInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
JDBCInterpreter.class.getName() + this.hashCode());
String schedulerName = JDBCInterpreter.class.getName() + this.hashCode();
return isConcurrentExecution() ?
SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName, 10)
: SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName);
}
@Override
@ -454,5 +459,17 @@ public class JDBCInterpreter extends Interpreter {
return Integer.valueOf(
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
}
boolean isConcurrentExecution() {
return Boolean.valueOf(getProperty(CONCURRENT_EXECUTION_KEY));
}
int getMaxConcurrentConnection() {
try {
return Integer.valueOf(getProperty(CONCURRENT_EXECUTION_COUNT));
} catch (Exception e) {
return 10;
}
}
}

12
jdbc/src/main/resources/interpreter-setting.json
View file

@ -33,6 +33,18 @@
"propertyName": "common.max_count",
"defaultValue": "1000",
"description": "Max number of SQL result to display."
},
"zeppelin.jdbc.concurrent.use": {
"envName": null,
"propertyName": "zeppelin.jdbc.concurrent.use",
"defaultValue": "true",
"description": "Use parallel scheduler"
},
"zeppelin.jdbc.concurrent.max_connection": {
"envName": null,
"propertyName": "zeppelin.jdbc.concurrent.max_connection",
"defaultValue": "10",
"description": "Number of concurrent execution"
}
}
}

28
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -22,6 +22,8 @@ import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
@ -32,6 +34,9 @@ import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.jdbc.JDBCInterpreter;
import org.apache.zeppelin.scheduler.FIFOScheduler;
import org.apache.zeppelin.scheduler.ParallelScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.junit.Before;
import org.junit.Test;
@ -200,4 +205,27 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message());
}
@Test
public void concurrentSettingTest() {
Properties properties = new Properties();
properties.setProperty("zeppelin.jdbc.concurrent.use", "true");
properties.setProperty("zeppelin.jdbc.concurrent.max_connection", "10");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
assertTrue(jdbcInterpreter.isConcurrentExecution());
assertEquals(10, jdbcInterpreter.getMaxConcurrentConnection());
Scheduler scheduler = jdbcInterpreter.getScheduler();
assertTrue(scheduler instanceof ParallelScheduler);
properties.clear();
properties.setProperty("zeppelin.jdbc.concurrent.use", "false");
jdbcInterpreter = new JDBCInterpreter(properties);
assertFalse(jdbcInterpreter.isConcurrentExecution());
scheduler = jdbcInterpreter.getScheduler();
assertTrue(scheduler instanceof FIFOScheduler);
}
}

24
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
View file

@ -20,7 +20,6 @@ package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
@ -39,12 +38,8 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Properties;
/***
@ -64,11 +59,11 @@ public class LivyHelper {
public Integer createSession(InterpreterContext context, String kind) throws Exception {
try {
Map<String, String> conf = new HashMap<String, String>();
Iterator<Entry<Object, Object>> it = property.entrySet().iterator();
while (it.hasNext()) {
Entry<Object, Object> pair = it.next();
if (pair.getKey().toString().startsWith("livy.spark.") &&
if (pair.getKey().toString().startsWith("livy.spark.") &&
!pair.getValue().toString().isEmpty())
conf.put(pair.getKey().toString().substring(5), pair.getValue().toString());
}
@ -76,12 +71,12 @@ public class LivyHelper {
String confData = gson.toJson(conf);
String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions",
"POST",
"POST",
"{" +
"\"kind\": \"" + kind + "\", " +
"\"conf\": " + confData + ", " +
"\"proxyUser\": " + context.getAuthenticationInfo().getUser() +
"}",
"\"conf\": " + confData + ", " +
"\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() +
"\"}",
context.getParagraphId()
);
@ -96,9 +91,8 @@ public class LivyHelper {
LOGGER.error(String.format("sessionId:%s state is %s",
jsonMap.get("id"), jsonMap.get("state")));
Thread.sleep(1000);
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId,
"GET", null,
context.getParagraphId());
json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" +
sessionId, "GET", null, context.getParagraphId());
jsonMap = (Map<Object, Object>) gson.fromJson(json,
new TypeToken<Map<Object, Object>>() {
}.getType());

10
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
View file

@ -37,16 +37,6 @@ public class LivyPySparkInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivyPySparkInterpreter.class);
static {
Interpreter.register(
"pyspark",
"livy",
LivyPySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
protected LivyHelper livyHelper;

29
livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
View file

@ -34,38 +34,9 @@ import java.util.Properties;
*/
public class LivySparkInterpreter extends Interpreter {
static String DEFAULT_URL = "http://localhost:8998";
static String LOCAL = "local[*]";
Logger LOGGER = LoggerFactory.getLogger(LivySparkInterpreter.class);
private LivyOutputStream out;
static {
Interpreter.register(
"spark",
"livy",
LivySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.")
.add("livy.spark.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
.add("livy.spark.driver.cores", "", "Driver cores. ex) 1, 2")
.add("livy.spark.driver.memory", "", "Driver memory. ex) 512m, 32g")
.add("livy.spark.executor.instances", "", "Executor instances. ex) 1, 4")
.add("livy.spark.executor.cores", "", "Num cores per executor. ex) 1, 4")
.add("livy.spark.executor.memory", "",
"Executor memory per worker instance. ex) 512m, 32g")
.add("livy.spark.dynamicAllocation.enabled", "", "Use dynamic resource allocation")
.add("livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", "",
"Remove an executor which has cached data blocks")
.add("livy.spark.dynamicAllocation.minExecutors", "",
"Lower bound for the number of executors if dynamic allocation is enabled. ")
.add("livy.spark.dynamicAllocation.initialExecutors", "",
"Initial number of executors to run if dynamic allocation is enabled. ")
.add("livy.spark.dynamicAllocation.maxExecutors", "",
"Upper bound for the number of executors if dynamic allocation is enabled. ")
.build()
);
}
protected static Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;

10
livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
View file

@ -37,16 +37,6 @@ public class LivySparkRInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkRInterpreter.class);
static {
Interpreter.register(
"sparkr",
"livy",
LivySparkRInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;

16
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -35,20 +35,6 @@ import java.util.Properties;
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
static String DEFAULT_MAX_RESULT = "1000";
static {
Interpreter.register(
"sql",
"livy",
LivySparkSQLInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.livy.spark.maxResult",
DEFAULT_MAX_RESULT,
"Max number of SparkSQL result to display.")
.build()
);
}
protected Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
@ -94,7 +80,7 @@ public class LivySparkSQLInterpreter extends Interpreter {
line.replaceAll("\"", "\\\\\"")
.replaceAll("\\n", " ")
+ "\").show(" +
property.get("zeppelin.livy.spark.maxResult") + ")",
property.get("livy.spark.sql.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {

97
livy/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,97 @@
[
{
"group": "livy",
"name": "spark",
"className": "org.apache.zeppelin.livy.LivySparkInterpreter",
"properties": {
"zeppelin.livy.url": {
"envName": "ZEPPELIN_LIVY_HOST_URL",
"propertyName": "zeppelin.livy.url",
"defaultValue": "http://localhost:8998",
"description": "The URL for Livy Server."
},
"livy.spark.master": {
"propertyName": "livy.spark.master",
"defaultValue": "local[*]",
"description": "Spark master uri. ex) spark://masterhost:7077"
},
"livy.spark.driver.cores": {
"propertyName": "livy.spark.driver.cores",
"defaultValue": "",
"description": "Driver cores. ex) 1, 2"
},
"livy.spark.driver.memory": {
"propertyName": "livy.spark.driver.memory",
"defaultValue": "",
"description": "Driver memory. ex) 512m, 32g"
},
"livy.spark.executor.instances": {
"propertyName": "livy.spark.executor.instances",
"defaultValue": "",
"description": "Executor instances. ex) 1, 4"
},
"livy.spark.executor.cores": {
"propertyName": "livy.spark.executor.cores",
"defaultValue": "",
"description": "Num cores per executor. ex) 1, 4"
},
"livy.spark.executor.memory": {
"propertyName": "livy.spark.executor.memory",
"defaultValue": "",
"description": "Executor memory per worker instance. ex) 512m, 32g"
},
"livy.spark.dynamicAllocation.enabled": {
"propertyName": "livy.spark.dynamicAllocation.enabled",
"defaultValue": "",
"description": "Use dynamic resource allocation"
},
"livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": {
"propertyName": "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout",
"defaultValue": "",
"description": "Remove an executor which has cached data blocks"
},
"livy.spark.dynamicAllocation.minExecutors": {
"propertyName": "livy.spark.dynamicAllocation.minExecutors",
"defaultValue": "",
"description": "Lower bound for the number of executors if dynamic allocation is enabled."
},
"livy.spark.dynamicAllocation.initialExecutors": {
"propertyName": "livy.spark.dynamicAllocation.initialExecutors",
"defaultValue": "",
"description": "Initial number of executors to run if dynamic allocation is enabled."
},
"livy.spark.dynamicAllocation.maxExecutors": {
"propertyName": "livy.spark.dynamicAllocation.maxExecutors",
"defaultValue": "",
"description": "Upper bound for the number of executors if dynamic allocation is enabled."
}
}
},
{
"group": "livy",
"name": "sql",
"className": "org.apache.zeppelin.livy.LivySparkSQLInterpreter",
"properties": {
"zeppelin.livy.spark.sql.maxResult": {
"envName": "ZEPPELIN_LIVY_MAXRESULT",
"propertyName": "zeppelin.livy.spark.sql.maxResult",
"defaultValue": "1000",
"description": "Max number of SparkSQL result to display."
}
}
},
{
"group": "livy",
"name": "pyspark",
"className": "org.apache.zeppelin.livy.LivyPySparkInterpreter",
"properties": {
}
},
{
"group": "livy",
"name": "sparkr",
"className": "org.apache.zeppelin.livy.LivySparkRInterpreter",
"properties": {
}
}
]

262
notebook/2BQA35CJZ/note.json Normal file
View file

File diff suppressed because one or more lines are too long

149
phoenix/pom.xml
View file

@ -1,149 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-phoenix</artifactId>
<packaging>jar</packaging>
<version>0.6.0-SNAPSHOT</version>
<name>Zeppelin: Apache Phoenix Interpreter</name>
<description>Zeppelin interprter for Apache Phoenix</description>
<url>http://zeppelin.apache.org</url>
<properties>
<phoenix.version>4.4.0-HBase-1.0</phoenix.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>sqlline</groupId>
<artifactId>sqlline</artifactId>
<version>1.1.9</version>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
<version>${phoenix.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mockrunner</groupId>
<artifactId>mockrunner-jdbc</artifactId>
<version>1.0.8</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/phoenix</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/phoenix</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

240
phoenix/src/main/java/org/apache/zeppelin/phoenix/PhoenixInterpreter.java
View file

@ -1,240 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.phoenix;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Phoenix interpreter for Zeppelin.
*/
public class PhoenixInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(PhoenixInterpreter.class);
private static final String EXPLAIN_PREDICATE = "EXPLAIN ";
private static final String UPDATE_HEADER = "UPDATES ";
private static final String WS = " ";
private static final String NEWLINE = "\n";
private static final String TAB = "\t";
private static final String TABLE_MAGIC_TAG = "%table ";
static final String PHOENIX_JDBC_URL = "phoenix.jdbc.url";
static final String PHOENIX_JDBC_USER = "phoenix.user";
static final String PHOENIX_JDBC_PASSWORD = "phoenix.password";
static final String PHOENIX_MAX_RESULT = "phoenix.max.result";
static final String PHOENIX_JDBC_DRIVER_NAME = "phoenix.driver.name";
static final String DEFAULT_JDBC_URL = "jdbc:phoenix:localhost:2181:/hbase-unsecure";
static final String DEFAULT_JDBC_USER = "";
static final String DEFAULT_JDBC_PASSWORD = "";
static final String DEFAULT_MAX_RESULT = "1000";
static final String DEFAULT_JDBC_DRIVER_NAME = "org.apache.phoenix.jdbc.PhoenixDriver";
private Connection jdbcConnection;
private Statement currentStatement;
private Exception exceptionOnConnect;
private int maxResult;
static {
Interpreter.register(
"sql",
"phoenix",
PhoenixInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(PHOENIX_JDBC_URL, DEFAULT_JDBC_URL, "Phoenix JDBC connection string")
.add(PHOENIX_JDBC_USER, DEFAULT_JDBC_USER, "The Phoenix user")
.add(PHOENIX_JDBC_PASSWORD, DEFAULT_JDBC_PASSWORD, "The password for the Phoenix user")
.add(PHOENIX_MAX_RESULT, DEFAULT_MAX_RESULT, "Max number of SQL results to display.")
.add(PHOENIX_JDBC_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME, "Phoenix Driver classname.")
.build()
);
}
public PhoenixInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
logger.info("Jdbc open connection called!");
close();
try {
Class.forName(getProperty(PHOENIX_JDBC_DRIVER_NAME));
maxResult = Integer.valueOf(getProperty(PHOENIX_MAX_RESULT));
jdbcConnection = DriverManager.getConnection(
getProperty(PHOENIX_JDBC_URL),
getProperty(PHOENIX_JDBC_USER),
getProperty(PHOENIX_JDBC_PASSWORD)
);
exceptionOnConnect = null;
logger.info("Successfully created Jdbc connection");
}
catch (ClassNotFoundException | SQLException e) {
logger.error("Cannot open connection", e);
exceptionOnConnect = e;
}
}
@Override
public void close() {
logger.info("Jdbc close connection called!");
try {
if (getJdbcConnection() != null) {
getJdbcConnection().close();
}
} catch (SQLException e) {
logger.error("Cannot close connection", e);
}
finally {
exceptionOnConnect = null;
}
}
private String clean(boolean isExplain, String str){
return (isExplain || str == null) ? str : str.replace(TAB, WS).replace(NEWLINE, WS);
}
private InterpreterResult executeSql(String sql) {
try {
if (exceptionOnConnect != null) {
return new InterpreterResult(Code.ERROR, exceptionOnConnect.getMessage());
}
currentStatement = getJdbcConnection().createStatement();
boolean isExplain = StringUtils.containsIgnoreCase(sql, EXPLAIN_PREDICATE);
StringBuilder msg = (isExplain) ? new StringBuilder() : new StringBuilder(TABLE_MAGIC_TAG);
ResultSet res = null;
try {
boolean hasResult = currentStatement.execute(sql);
if (hasResult){ //If query had results
res = currentStatement.getResultSet();
//Append column names
ResultSetMetaData md = res.getMetaData();
String row = clean(isExplain, md.getColumnName(1));
for (int i = 2; i < md.getColumnCount() + 1; i++)
row += TAB + clean(isExplain, md.getColumnName(i));
msg.append(row + NEWLINE);
//Append rows
int rowCount = 0;
while (res.next() && rowCount < getMaxResult()) {
row = clean(isExplain, res.getString(1));
for (int i = 2; i < md.getColumnCount() + 1; i++)
row += TAB + clean(isExplain, res.getString(i));
msg.append(row + NEWLINE);
rowCount++;
}
}
else { // May have been upsert or DDL
msg.append(UPDATE_HEADER + NEWLINE +
"Rows affected: " + currentStatement.getUpdateCount()
+ NEWLINE);
}
} finally {
try {
if (res != null) res.close();
getJdbcConnection().commit();
currentStatement.close();
} finally {
currentStatement = null;
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
}
catch (SQLException ex) {
logger.error("Can not run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run SQL command '" + cmd + "'");
return executeSql(cmd);
}
@Override
public void cancel(InterpreterContext context) {
if (currentStatement != null) {
try {
currentStatement.cancel();
}
catch (SQLException ex) {
}
finally {
currentStatement = null;
}
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
PhoenixInterpreter.class.getName() + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}
public Connection getJdbcConnection() {
return jdbcConnection;
}
public int getMaxResult() {
return maxResult;
}
}

226
phoenix/src/test/java/org/apache/zeppelin/phoenix/PhoenixInterpreterTest.java
View file

@ -1,226 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.phoenix;
import static org.apache.zeppelin.phoenix.PhoenixInterpreter.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import com.mockrunner.jdbc.StatementResultSetHandler;
import com.mockrunner.mock.jdbc.MockConnection;
import com.mockrunner.mock.jdbc.MockResultSet;
/**
* Phoenix interpreter unit tests
*/
public class PhoenixInterpreterTest extends BasicJDBCTestCaseAdapter {
private PhoenixInterpreter phoenixInterpreter = null;
private MockResultSet result = null;
@Before
public void beforeTest() {
MockConnection connection = getJDBCMockObjectFactory().getMockConnection();
StatementResultSetHandler statementHandler = connection.getStatementResultSetHandler();
result = statementHandler.createResultSet();
statementHandler.prepareGlobalResultSet(result);
Properties properties = new Properties();
properties.put(PHOENIX_JDBC_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME);
properties.put(PHOENIX_JDBC_URL, DEFAULT_JDBC_URL);
properties.put(PHOENIX_JDBC_USER, DEFAULT_JDBC_USER);
properties.put(PHOENIX_JDBC_PASSWORD, DEFAULT_JDBC_PASSWORD);
properties.put(PHOENIX_MAX_RESULT, DEFAULT_MAX_RESULT);
phoenixInterpreter = spy(new PhoenixInterpreter(properties));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(connection);
}
@Test
public void testOpenCommandIdempotency() throws SQLException {
// Ensure that an attempt to open new connection will clean any remaining connections
phoenixInterpreter.open();
phoenixInterpreter.open();
phoenixInterpreter.open();
verify(phoenixInterpreter, times(3)).open();
verify(phoenixInterpreter, times(3)).close();
}
@Test
public void testDefaultProperties() throws SQLException {
PhoenixInterpreter phoenixInterpreter = new PhoenixInterpreter(new Properties());
assertEquals(DEFAULT_JDBC_DRIVER_NAME,
phoenixInterpreter.getProperty(PHOENIX_JDBC_DRIVER_NAME));
assertEquals(DEFAULT_JDBC_URL, phoenixInterpreter.getProperty(PHOENIX_JDBC_URL));
assertEquals(DEFAULT_JDBC_USER, phoenixInterpreter.getProperty(PHOENIX_JDBC_USER));
assertEquals(DEFAULT_JDBC_PASSWORD,
phoenixInterpreter.getProperty(PHOENIX_JDBC_PASSWORD));
assertEquals(DEFAULT_MAX_RESULT, phoenixInterpreter.getProperty(PHOENIX_MAX_RESULT));
}
@Test
public void testConnectionClose() throws SQLException {
PhoenixInterpreter phoenixInterpreter = spy(new PhoenixInterpreter(new Properties()));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
phoenixInterpreter.close();
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
verifyConnectionClosed();
}
@Test
public void testStatementCancel() throws SQLException {
PhoenixInterpreter phoenixInterpreter = spy(new PhoenixInterpreter(new Properties()));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
phoenixInterpreter.cancel(null);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
assertFalse("Cancel operation should not close the connection", phoenixInterpreter
.getJdbcConnection().isClosed());
}
@Test
public void testSelectQuery() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\nval12\tval22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryMaxResult() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryWithSpecialCharacters() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("co\tl1", new String[] {"val11", "va\tl1\n2"});
result.addColumn("co\nl2", new String[] {"v\nal21", "val\t22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("co l1\tco l2\nval11\tv al21\nva l1 2\tval 22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQuery() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("col1\nval11\nval12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQueryWithSpecialCharachters() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("co\tl\n1", new String[] {"va\nl11", "va\tl\n12"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("co\tl\n1\nva\nl11\nva\tl\n12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
}

8
pom.xml
View file

@ -73,7 +73,6 @@
<module>shell</module>
<module>livy</module>
<module>hbase</module>
<module>phoenix</module>
<module>postgresql</module>
<module>jdbc</module>
<module>file</module>
@ -95,7 +94,7 @@
<log4j.version>1.2.17</log4j.version>
<libthrift.version>0.9.2</libthrift.version>
<gson.version>2.2</gson.version>
<guava.version>15.0</guava.version>
<guava.version>19.0</guava.version>
<jetty.version>9.2.15.v20160210</jetty.version>
<PermGen>64m</PermGen>
@ -212,6 +211,11 @@
<artifactId>shiro-web</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-config-core</artifactId>
<version>1.2.3</version>
</dependency>
</dependencies>
</dependencyManagement>

42
python/README.md Normal file
View file

@ -0,0 +1,42 @@
# Overview
Python interpreter for Apache Zeppelin
# Architecture
Current interpreter implementation spawns new system python process through `ProcessBuilder` and re-directs it's stdin\strout to Zeppelin
# Details
- **Py4j support**
[Py4j](https://www.py4j.org/) enables Python programs to dynamically access Java objects in a JVM.
It is required in order to use Zeppelin [dynamic forms](http://zeppelin.apache.org/docs/0.6.0-SNAPSHOT/manual/dynamicform.html) feature.
- bootstrap process
Interpreter environment is setup with thex [bootstrap.py](https://github.com/apache/zeppelin/blob/master/python/src/main/resources/bootstrap.py)
It defines `help()` and `z` convenience functions
### Dev prerequisites
* Python 2 or 3 installed with py4j (0.9.2) and matplotlib (1.31 or later) installed on each
* Tests only checks the interpreter logic and starts any Python process! Python process is mocked with a class that simply output it input.
* Code wrote in `bootstrap.py` and `bootstrap_input.py` should always be Python 2 and 3 compliant.
* Use PEP8 convention for python code.
### Technical overview
* When interpreter is starting it launches a python process inside a Java ProcessBuilder. Python is started with -i (interactive mode) and -u (unbuffered stdin, stdout and stderr) options. Thus the interpreter has a "sleeping" python process.
* Interpreter sends command to python with a Java `outputStreamWiter` and read from an `InputStreamReader`. To know when stop reading stdout, interpreter sends `print "*!?flush reader!?*"`after each command and reads stdout until he receives back the `*!?flush reader!?*`.
* When interpreter is starting, it sends some Python code (bootstrap.py and bootstrap_input.py) to initialize default behavior and functions (`help(), z.input()...`). bootstrap_input.py is sent only if py4j library is detected inside Python process.
* [Py4J](https://www.py4j.org/) python and java libraries is used to load Input zeppelin Java class into the python process (make java code with python code !). Therefore the interpreter can directly create Zeppelin input form inside the Python process (and eventually with some python variable already defined). JVM opens a random open port to be accessible from python process.
* JavaBuilder can't send SIGINT signal to interrupt paragraph execution. Therefore interpreter directly send a `kill SIGINT PID` to python process to interrupt execution. Python process catch SIGINT signal with some code defined in bootstrap.py
* Matplotlib display feature is made with SVG export (in string) and then displays it with html code.

45
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -53,11 +53,12 @@ public class PythonInterpreter extends Interpreter {
private Integer port;
private GatewayServer gatewayServer;
PythonProcess process = null;
private long pythonPid;
private Boolean py4J = false;
private InterpreterContext context;
PythonProcess process = null;
static {
Interpreter.register(
"python",
@ -70,17 +71,13 @@ public class PythonInterpreter extends Interpreter {
);
}
public PythonInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
logger.info("Starting Python interpreter .....");
logger.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
process = getPythonProcess();
@ -117,13 +114,10 @@ public class PythonInterpreter extends Interpreter {
"initialize Zeppelin inputs in python process", e);
}
}
}
@Override
public void close() {
logger.info("closing Python interpreter .....");
try {
if (process != null) {
@ -135,13 +129,10 @@ public class PythonInterpreter extends Interpreter {
} catch (IOException e) {
logger.error("Can't close the interpreter", e);
}
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
this.context = contextInterpreter;
String output = sendCommandToPython(cmd);
@ -149,7 +140,6 @@ public class PythonInterpreter extends Interpreter {
.replaceAll("\\.\\.\\.", "").trim());
}
@Override
public void cancel(InterpreterContext context) {
try {
@ -171,8 +161,8 @@ public class PythonInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
PythonInterpreter.class.getName() + this.hashCode(), 10);
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
PythonInterpreter.class.getName() + this.hashCode());
}
@Override
@ -181,10 +171,11 @@ public class PythonInterpreter extends Interpreter {
}
public PythonProcess getPythonProcess() {
if (process == null)
if (process == null) {
return new PythonProcess(getProperty(ZEPPELIN_PYTHON));
else
} else {
return process;
}
}
private Job getRunningJob(String paragraphId) {
@ -193,6 +184,7 @@ public class PythonInterpreter extends Interpreter {
for (Job job : jobsRunning) {
if (job.getId().equals(paragraphId)) {
foundJob = job;
break;
}
}
return foundJob;
@ -200,21 +192,17 @@ public class PythonInterpreter extends Interpreter {
private String sendCommandToPython(String cmd) {
String output = "";
logger.info("Sending : \n " + cmd);
logger.info("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 120) + "..." : cmd));
try {
output = process.sendAndGetResult(cmd);
} catch (IOException e) {
logger.error("Error when sending commands to python process", e);
}
return output;
}
private void bootStrapInterpreter(String file) throws IOException {
BufferedReader bootstrapReader = new BufferedReader(
new InputStreamReader(
PythonInterpreter.class.getResourceAsStream(file)));
@ -227,30 +215,25 @@ public class PythonInterpreter extends Interpreter {
if (py4J && port != null && port != -1) {
bootstrapCode = bootstrapCode.replaceAll("\\%PORT\\%", port.toString());
}
logger.info("Bootstrap python interpreter with \n " + bootstrapCode);
logger.info("Bootstrap python interpreter with code from \n " + file);
sendCommandToPython(bootstrapCode);
}
public GUI getGui() {
return context.getGui();
}
public Integer getPy4JPort() {
return port;
}
public Boolean isPy4jInstalled() {
String output = sendCommandToPython("\n\nimport py4j\n");
if (output.contains("ImportError"))
if (output.contains("ImportError")) {
return false;
else return true;
} else {
return true;
}
}
private int findRandomOpenPortOnAllLocalInterfaces() {

18
python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
View file

@ -33,15 +33,15 @@ import java.lang.reflect.Field;
* Object encapsulated interactive
* Python process (REPL) used by python interpreter
*/
public class PythonProcess {
Logger logger = LoggerFactory.getLogger(PythonProcess.class);
InputStream stdout;
OutputStream stdin;
BufferedWriter writer;
BufferedReader reader;
Process process;
private String binPath;
private long pid;
@ -64,22 +64,17 @@ public class PythonProcess {
logger.warn("Can't find python pid process", e);
pid = -1;
}
}
public void close() throws IOException {
process.destroy();
reader.close();
writer.close();
stdin.close();
stdout.close();
}
public void interrupt() throws IOException {
if (pid > -1) {
logger.info("Sending SIGINT signal to PID : " + pid);
Runtime.getRuntime().exec("kill -SIGINT " + pid);
@ -87,12 +82,9 @@ public class PythonProcess {
logger.warn("Non UNIX/Linux system, close the interpreter");
close();
}
}
public String sendAndGetResult(String cmd) throws IOException {
writer.write(cmd + "\n\n");
writer.write("print (\"*!?flush reader!?*\")\n\n");
writer.flush();
@ -106,18 +98,13 @@ public class PythonProcess {
output += "Syntax error ! ";
break;
}
output += "\r" + line + "\n";
}
return output;
}
private long findPid() throws NoSuchFieldException, IllegalAccessException {
long pid = -1;
if (process.getClass().getName().equals("java.lang.UNIXProcess")) {
Field f = process.getClass().getDeclaredField("pid");
f.setAccessible(true);
@ -130,4 +117,5 @@ public class PythonProcess {
public long getPid() {
return pid;
}
}

9
python/src/main/resources/bootstrap.py
View file

@ -14,11 +14,12 @@
# limitations under the License.
# PYTHON 2 / 3 comptability :
# bootstrap.py must be runnable with Python 2 and 3
# bootstrap.py must be runnable with Python 2 or 3
# Remove interactive mode displayhook
import sys
import signal
try:
import StringIO as io
except ImportError:
@ -26,14 +27,12 @@ except ImportError:
sys.displayhook = lambda x: None
def intHandler(signum, frame): # Set the signal handler
print ("Paragraph interrupted")
raise KeyboardInterrupt()
signal.signal(signal.SIGINT, intHandler)
def help():
print ('%html')
print ('<h2>Python Interpreter help</h2>')
@ -72,9 +71,7 @@ plt.close()
print('''<pre>zeppelin_show(plt,width='50px')
zeppelin_show(plt,height='150px') </pre></div>''')
# Matplotlib show function
def zeppelin_show(p, width="0", height="0"):
img = io.StringIO()
p.savefig(img, format='svg')
@ -88,10 +85,8 @@ def zeppelin_show(p, width="0", height="0"):
style += 'height:'+height
print("%html <div style='" + style + "'>" + img.read() + "<div>")
# If py4j is detected, these class will be override
# with the implementation in bootstrap_input.py
class PyZeppelinContext():
errorMsg = "You must install py4j Python module " \
"(pip install py4j) to use Zeppelin dynamic forms features"

7
python/src/main/resources/bootstrap_input.py
View file

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from py4j.java_gateway import JavaGateway
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
@ -21,14 +20,16 @@ client = GatewayClient(port=%PORT%)
gateway = JavaGateway(client)
java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
class PyZeppelinContext():
paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
javaList = gateway.jvm.java.util.ArrayList
def __init__(self, zc):
self.z = zc
def input(self, name, defaultValue=""):
return self.z.getGui().input(name, defaultValue)
def select(self, name, options, defaultValue=""):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
@ -36,6 +37,7 @@ class PyZeppelinContext():
javaOptions[i] = self.paramOption(tuple[0], tuple[1])
i += 1
return self.z.getGui().select(name, defaultValue, javaOptions)
def checkbox(self, name, options, defaultChecked=[]):
javaOptions = gateway.new_array(self.paramOption, len(options))
i = 0
@ -47,5 +49,4 @@ class PyZeppelinContext():
javaDefaultCheck.append(check)
return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
z = PyZeppelinContext(gateway.entry_point)

1
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -40,7 +40,6 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Ignite (http://ignite.apache.org/)
(Apache 2.0) Apache Kylin (http://kylin.apache.org/)
(Apache 2.0) Apache Lens (http://lens.apache.org/)
(Apache 2.0) Apache Phoenix (http://phoenix.apache.org/)
(Apache 2.0) Apache Flink (http://flink.apache.org/)
(Apache 2.0) Apache Thrift (http://thrift.apache.org/)
(Apache 2.0) Apache Lucene (https://lucene.apache.org/)

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ByteBufferInputStream.java
View file

@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
/**
* InputStream from bytebuffer
*/

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/user/AuthenticationInfo.java
View file

@ -28,6 +28,10 @@ public class AuthenticationInfo {
public AuthenticationInfo() {}
public AuthenticationInfo(String user) {
this.user = user;
}
/***
*
* @param user

48
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -46,6 +46,7 @@ import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.socket.NotebookServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.utils.SecurityUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
@ -158,7 +159,8 @@ public class NotebookRestApi {
notebookAuthorization.getOwners(noteId),
notebookAuthorization.getReaders(noteId),
notebookAuthorization.getWriters(noteId));
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse<>(Status.OK).build();
}
@ -224,7 +226,8 @@ public class NotebookRestApi {
@Path("/")
@ZeppelinApi
public Response getNotebookList() throws IOException {
List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false, subject);
return new JsonResponse<>(Status.OK, "", notesInfo ).build();
}
@ -266,7 +269,8 @@ public class NotebookRestApi {
@Path("import")
@ZeppelinApi
public Response importNotebook(String req) throws IOException {
Note newNote = notebook.importNote(req, null);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.importNote(req, null, subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
@ -283,7 +287,8 @@ public class NotebookRestApi {
LOG.info("Create new notebook by JSON {}" , message);
NewNotebookRequest request = gson.fromJson(message,
NewNotebookRequest.class);
Note note = notebook.createNote();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note note = notebook.createNote(subject);
List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
if (initialParagraphs != null) {
for (NewParagraphRequest paragraphRequest : initialParagraphs) {
@ -297,10 +302,11 @@ public class NotebookRestApi {
if (noteName.isEmpty()) {
noteName = "Note " + note.getId();
}
note.setName(noteName);
note.persist();
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList();
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", note.getId() ).build();
}
@ -315,13 +321,15 @@ public class NotebookRestApi {
@ZeppelinApi
public Response deleteNote(@PathParam("notebookId") String notebookId) throws IOException {
LOG.info("Delete notebook {} ", notebookId);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
if (!(notebookId.isEmpty())) {
Note note = notebook.getNote(notebookId);
if (note != null) {
notebook.removeNote(notebookId);
notebook.removeNote(notebookId, subject);
}
}
notebookServer.broadcastNoteList();
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.OK, "").build();
}
@ -340,9 +348,10 @@ public class NotebookRestApi {
NewNotebookRequest request = gson.fromJson(message,
NewNotebookRequest.class);
String newNoteName = request.getName();
Note newNote = notebook.cloneNote(notebookId, newNoteName);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
notebookServer.broadcastNoteList();
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
@ -376,7 +385,8 @@ public class NotebookRestApi {
p.setTitle(request.getTitle());
p.setText(request.getText());
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.CREATED, "", p.getId()).build();
}
@ -434,7 +444,8 @@ public class NotebookRestApi {
try {
note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true);
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.OK, "").build();
} catch (IndexOutOfBoundsException e) {
@ -466,8 +477,9 @@ public class NotebookRestApi {
return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.removeParagraph(paragraphId);
note.persist();
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.OK, "").build();
@ -574,7 +586,9 @@ public class NotebookRestApi {
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.setLastReplName(paragraph.getId());
note.persist(subject);
}
}
@ -700,7 +714,8 @@ public class NotebookRestApi {
public Response getJobListforNotebook() throws IOException, IllegalArgumentException {
LOG.info("Get notebook jobs for job manager");
List<Map<String, Object>> notebookJobs = notebook.getJobListforNotebook(false, 0);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
List<Map<String, Object>> notebookJobs = notebook.getJobListforNotebook(false, 0, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -724,7 +739,8 @@ public class NotebookRestApi {
LOG.info("Get updated notebook jobs lastUpdateTime {}", lastUpdateUnixTime);
List<Map<String, Object>> notebookJobs;
notebookJobs = notebook.getJobListforNotebook(false, lastUpdateUnixTime);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
notebookJobs = notebook.getJobListforNotebook(false, lastUpdateUnixTime, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());

20
zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java
View file

@ -22,8 +22,6 @@ import org.apache.shiro.realm.Realm;
import org.apache.shiro.realm.jdbc.JdbcRealm;
import org.apache.shiro.realm.ldap.JndiLdapRealm;
import org.apache.shiro.realm.text.IniRealm;
import org.apache.shiro.util.ThreadContext;
import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.server.JsonResponse;
@ -41,7 +39,6 @@ import java.util.*;
/**
* Zeppelin security rest api endpoint.
*
*/
@Path("/security")
@Produces("application/json")
@ -101,19 +98,16 @@ public class SecurityRestApi {
List<String> usersList = new ArrayList<>();
try {
GetUserList getUserListObj = new GetUserList();
DefaultWebSecurityManager defaultWebSecurityManager;
String key = ThreadContext.SECURITY_MANAGER_KEY;
defaultWebSecurityManager = (DefaultWebSecurityManager) ThreadContext.get(key);
Collection<Realm> realms = defaultWebSecurityManager.getRealms();
List realmsList = new ArrayList(realms);
for (int i = 0; i < realmsList.size(); i++) {
String name = ((Realm) realmsList.get(i)).getName();
Collection realmsList = SecurityUtils.getRealmsList();
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getName();
if (name.equals("iniRealm")) {
usersList.addAll(getUserListObj.getUserList((IniRealm) realmsList.get(i)));
usersList.addAll(getUserListObj.getUserList((IniRealm) realm));
} else if (name.equals("ldapRealm")) {
usersList.addAll(getUserListObj.getUserList((JndiLdapRealm) realmsList.get(i)));
usersList.addAll(getUserListObj.getUserList((JndiLdapRealm) realm));
} else if (name.equals("jdbcRealm")) {
usersList.addAll(getUserListObj.getUserList((JdbcRealm) realmsList.get(i)));
usersList.addAll(getUserListObj.getUserList((JdbcRealm) realm));
}
}

241
zeppelin-server/src/main/java/org/apache/zeppelin/server/ActiveDirectoryGroupRealm.java Normal file
View file

@ -0,0 +1,241 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.server;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.realm.Realm;
import org.apache.shiro.realm.ldap.AbstractLdapRealm;
import org.apache.shiro.realm.ldap.LdapContextFactory;
import org.apache.shiro.realm.ldap.LdapUtils;
import org.apache.shiro.subject.PrincipalCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.LdapContext;
import java.util.*;
/**
* A {@link Realm} that authenticates with an active directory LDAP
* server to determine the roles for a particular user. This implementation
* queries for the user's groups and then maps the group names to roles using the
* {@link #groupRolesMap}.
*
* @since 0.1
*/
public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
private static final Logger log = LoggerFactory.getLogger(ActiveDirectoryGroupRealm.class);
private static final String ROLE_NAMES_DELIMETER = ",";
/*--------------------------------------------
| I N S T A N C E V A R I A B L E S |
============================================*/
/**
* Mapping from fully qualified active directory
* group names (e.g. CN=Group,OU=Company,DC=MyDomain,DC=local)
* as returned by the active directory LDAP server to role names.
*/
private Map<String, String> groupRolesMap;
/*--------------------------------------------
| C O N S T R U C T O R S |
============================================*/
public void setGroupRolesMap(Map<String, String> groupRolesMap) {
this.groupRolesMap = groupRolesMap;
}
/*--------------------------------------------
| M E T H O D S |
============================================*/
/**
* Builds an {@link AuthenticationInfo} object by querying the active directory LDAP context for
* the specified username. This method binds to the LDAP server using the provided username
* and password - which if successful, indicates that the password is correct.
* <p/>
* This method can be overridden by subclasses to query the LDAP server in a more complex way.
*
* @param token the authentication token provided by the user.
* @param ldapContextFactory the factory used to build connections to the LDAP server.
* @return an {@link AuthenticationInfo} instance containing information retrieved from LDAP.
* @throws NamingException if any LDAP errors occur during the search.
*/
protected AuthenticationInfo queryForAuthenticationInfo(
AuthenticationToken token, LdapContextFactory ldapContextFactory) throws NamingException {
UsernamePasswordToken upToken = (UsernamePasswordToken) token;
// Binds using the username and password provided by the user.
LdapContext ctx = null;
try {
String userPrincipalName = upToken.getUsername();
if (userPrincipalName == null) {
return null;
}
if (this.principalSuffix != null) {
userPrincipalName = upToken.getUsername() + this.principalSuffix;
}
ctx = ldapContextFactory.getLdapContext(
userPrincipalName, upToken.getPassword());
} finally {
LdapUtils.closeContext(ctx);
}
return buildAuthenticationInfo(upToken.getUsername(), upToken.getPassword());
}
protected AuthenticationInfo buildAuthenticationInfo(String username, char[] password) {
return new SimpleAuthenticationInfo(username, password, getName());
}
/**
* Builds an {@link org.apache.shiro.authz.AuthorizationInfo} object by querying the active
* directory LDAP context for the groups that a user is a member of. The groups are then
* translated to role names by using the configured {@link #groupRolesMap}.
* <p/>
* This implementation expects the <tt>principal</tt> argument to be a String username.
* <p/>
* Subclasses can override this method to determine authorization data (roles, permissions, etc)
* in a more complex way. Note that this default implementation does not support permissions,
* only roles.
*
* @param principals the principal of the Subject whose account is being retrieved.
* @param ldapContextFactory the factory used to create LDAP connections.
* @return the AuthorizationInfo for the given Subject principal.
* @throws NamingException if an error occurs when searching the LDAP server.
*/
protected AuthorizationInfo queryForAuthorizationInfo(
PrincipalCollection principals,
LdapContextFactory ldapContextFactory) throws NamingException {
String username = (String) getAvailablePrincipal(principals);
// Perform context search
LdapContext ldapContext = ldapContextFactory.getSystemLdapContext();
Set<String> roleNames;
try {
roleNames = getRoleNamesForUser(username, ldapContext);
} finally {
LdapUtils.closeContext(ldapContext);
}
return buildAuthorizationInfo(roleNames);
}
protected AuthorizationInfo buildAuthorizationInfo(Set<String> roleNames) {
return new SimpleAuthorizationInfo(roleNames);
}
private Set<String> getRoleNamesForUser(String username, LdapContext ldapContext)
throws NamingException {
Set<String> roleNames = new LinkedHashSet<>();
SearchControls searchCtls = new SearchControls();
searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
String userPrincipalName = username;
if (principalSuffix != null) {
userPrincipalName += principalSuffix;
}
String searchFilter = "(&(objectClass=*)(userPrincipalName=" + userPrincipalName + "))";
Object[] searchArguments = new Object[]{userPrincipalName};
NamingEnumeration answer = ldapContext.search(searchBase, searchFilter, searchArguments,
searchCtls);
while (answer.hasMoreElements()) {
SearchResult sr = (SearchResult) answer.next();
if (log.isDebugEnabled()) {
log.debug("Retrieving group names for user [" + sr.getName() + "]");
}
Attributes attrs = sr.getAttributes();
if (attrs != null) {
NamingEnumeration ae = attrs.getAll();
while (ae.hasMore()) {
Attribute attr = (Attribute) ae.next();
if (attr.getID().equals("memberOf")) {
Collection<String> groupNames = LdapUtils.getAllAttributeValues(attr);
if (log.isDebugEnabled()) {
log.debug("Groups found for user [" + username + "]: " + groupNames);
}
Collection<String> rolesForGroups = getRoleNamesForGroups(groupNames);
roleNames.addAll(rolesForGroups);
}
}
}
}
return roleNames;
}
/**
* This method is called by the default implementation to translate Active Directory group names
* to role names. This implementation uses the {@link #groupRolesMap} to map group names to role
* names.
*
* @param groupNames the group names that apply to the current user.
* @return a collection of roles that are implied by the given role names.
*/
protected Collection<String> getRoleNamesForGroups(Collection<String> groupNames) {
Set<String> roleNames = new HashSet<String>(groupNames.size());
if (groupRolesMap != null) {
for (String groupName : groupNames) {
String strRoleNames = groupRolesMap.get(groupName);
if (strRoleNames != null) {
for (String roleName : strRoleNames.split(ROLE_NAMES_DELIMETER)) {
if (log.isDebugEnabled()) {
log.debug("User is member of group [" + groupName + "] so adding role [" +
roleName + "]");
}
roleNames.add(roleName);
}
}
}
}
return roleNames;
}
}

94
zeppelin-server/src/main/java/org/apache/zeppelin/server/LdapGroupRealm.java Normal file
View file

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.server;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.realm.ldap.JndiLdapRealm;
import org.apache.shiro.realm.ldap.LdapContextFactory;
import org.apache.shiro.subject.PrincipalCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.LdapContext;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;
/**
* Created for org.apache.zeppelin.server on 09/06/16.
*/
public class LdapGroupRealm extends JndiLdapRealm {
private static final Logger LOG = LoggerFactory.getLogger(LdapGroupRealm.class);
public AuthorizationInfo queryForAuthorizationInfo(
PrincipalCollection principals,
LdapContextFactory ldapContextFactory) throws NamingException {
String username = (String) getAvailablePrincipal(principals);
LdapContext ldapContext = ldapContextFactory.getSystemLdapContext();
Set<String> roleNames = getRoleNamesForUser(username, ldapContext, getUserDnTemplate());
return new SimpleAuthorizationInfo(roleNames);
}
public Set<String> getRoleNamesForUser(String username,
LdapContext ldapContext,
String userDnTemplate) throws NamingException {
try {
Set<String> roleNames = new LinkedHashSet<String>();
SearchControls searchCtls = new SearchControls();
searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);
String searchFilter = "(&(objectClass=groupOfNames)(member=" + userDnTemplate + "))";
Object[] searchArguments = new Object[]{username};
NamingEnumeration<?> answer = ldapContext.search(
String.valueOf(ldapContext.getEnvironment().get("ldap.searchBase")),
searchFilter,
searchArguments,
searchCtls);
while (answer.hasMoreElements()) {
SearchResult sr = (SearchResult) answer.next();
Attributes attrs = sr.getAttributes();
if (attrs != null) {
NamingEnumeration<?> ae = attrs.getAll();
while (ae.hasMore()) {
Attribute attr = (Attribute) ae.next();
if (attr.getID().equals("cn")) {
roleNames.add((String) attr.get());
}
}
}
}
return roleNames;
} catch (Exception e) {
LOG.error("Error", e);
}
return new HashSet<>();
}
}

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -45,6 +45,7 @@ import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.socket.NotebookServer;
import org.apache.zeppelin.user.Credentials;
import org.apache.zeppelin.utils.SecurityUtils;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
@ -255,6 +256,7 @@ public class ZeppelinServer extends Application {
webapp.setInitParameter("shiroConfigLocations",
new File(conf.getShiroPath()).toURI().toString());
SecurityUtils.initSecurityManager(conf.getShiroPath());
webapp.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/api/*",
EnumSet.allOf(DispatcherType.class));

127
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -149,17 +149,18 @@ public class NotebookServer extends WebSocketServlet implements
userAndRoles.addAll(roles);
}
}
AuthenticationInfo subject = new AuthenticationInfo(messagereceived.principal);
/** Lets be elegant here */
switch (messagereceived.op) {
case LIST_NOTES:
unicastNoteList(conn);
unicastNoteList(conn, subject);
break;
case RELOAD_NOTES_FROM_REPO:
broadcastReloadedNoteList();
broadcastReloadedNoteList(subject);
break;
case GET_HOME_NOTE:
sendHomeNote(conn, userAndRoles, notebook);
sendHomeNote(conn, userAndRoles, notebook, messagereceived);
break;
case GET_NOTE:
sendNote(conn, userAndRoles, notebook, messagereceived);
@ -221,7 +222,7 @@ public class NotebookServer extends WebSocketServlet implements
checkpointNotebook(conn, notebook, messagereceived);
break;
case LIST_NOTEBOOK_JOBS:
unicastNotebookJobInfo(conn);
unicastNotebookJobInfo(conn, messagereceived);
break;
case LIST_UPDATE_NOTEBOOK_JOBS:
unicastUpdateNotebookJobInfo(conn, messagereceived);
@ -373,9 +374,10 @@ public class NotebookServer extends WebSocketServlet implements
}
}
public void unicastNotebookJobInfo(NotebookSocket conn) throws IOException {
public void unicastNotebookJobInfo(NotebookSocket conn, Message fromMessage) throws IOException {
List<Map<String, Object>> notebookJobs = notebook().getJobListforNotebook(false, 0);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
List<Map<String, Object>> notebookJobs = notebook().getJobListforNotebook(false, 0, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -391,7 +393,8 @@ public class NotebookServer extends WebSocketServlet implements
long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue();
List<Map<String, Object>> notebookJobs;
notebookJobs = notebook().getJobListforNotebook(false, lastUpdateUnixTime);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebookJobs = notebook().getJobListforNotebook(false, lastUpdateUnixTime, subject);
Map<String, Object> response = new HashMap<>();
response.put("lastResponseUnixTime", System.currentTimeMillis());
@ -401,7 +404,9 @@ public class NotebookServer extends WebSocketServlet implements
.put("notebookRunningJobs", response)));
}
public List<Map<String, String>> generateNotebooksInfo(boolean needsReload) {
public List<Map<String, String>> generateNotebooksInfo(boolean needsReload,
AuthenticationInfo subject) {
Notebook notebook = notebook();
ZeppelinConfiguration conf = notebook.getConf();
@ -411,7 +416,7 @@ public class NotebookServer extends WebSocketServlet implements
if (needsReload) {
try {
notebook.reloadAllNotes();
notebook.reloadAllNotes(subject);
} catch (IOException e) {
LOG.error("Fail to reload notes from repository", e);
}
@ -438,28 +443,28 @@ public class NotebookServer extends WebSocketServlet implements
broadcast(note.id(), new Message(OP.NOTE).put("note", note));
}
public void broadcastNoteList() {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false);
public void broadcastNoteList(AuthenticationInfo subject) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject);
broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
}
public void unicastNoteList(NotebookSocket conn) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false);
public void unicastNoteList(NotebookSocket conn, AuthenticationInfo subject) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject);
unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn);
}
public void broadcastReloadedNoteList() {
List<Map<String, String>> notesInfo = generateNotebooksInfo(true);
public void broadcastReloadedNoteList(AuthenticationInfo subject) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(true, subject);
broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo));
}
void permissionError(NotebookSocket conn, String op, Set<String> userAndRoles,
void permissionError(NotebookSocket conn, String op,
String userName,
Set<String> userAndRoles,
Set<String> allowed) throws IOException {
LOG.info("Cannot {}. Connection readers {}. Allowed readers {}",
op, userAndRoles, allowed);
String userName = userAndRoles.iterator().next();
conn.send(serializeMessage(new Message(OP.AUTH_INFO).put("info",
"Insufficient privileges to " + op + " notebook.\n\n" +
"Allowed users or roles: " + allowed.toString() + "\n\n" +
@ -483,7 +488,8 @@ public class NotebookServer extends WebSocketServlet implements
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (note != null) {
if (!notebookAuthorization.isReader(noteId, userAndRoles)) {
permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId));
permissionError(conn, "read", fromMessage.principal, userAndRoles,
notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
@ -493,7 +499,7 @@ public class NotebookServer extends WebSocketServlet implements
}
private void sendHomeNote(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook) throws IOException {
Notebook notebook, Message fromMessage) throws IOException {
String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
Note note = null;
@ -504,7 +510,8 @@ public class NotebookServer extends WebSocketServlet implements
if (note != null) {
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isReader(noteId, userAndRoles)) {
permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId));
permissionError(conn, "read", fromMessage.principal,
userAndRoles, notebookAuthorization.getReaders(noteId));
return;
}
addConnectionToNote(note.id(), conn);
@ -532,7 +539,8 @@ public class NotebookServer extends WebSocketServlet implements
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "update", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "update", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
@ -545,9 +553,10 @@ public class NotebookServer extends WebSocketServlet implements
notebook.refreshCron(note.id());
}
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note.persist(subject);
broadcastNote(note);
broadcastNoteList();
broadcastNoteList(subject);
}
}
@ -568,7 +577,8 @@ public class NotebookServer extends WebSocketServlet implements
private void createNote(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook, Message message)
throws IOException {
Note note = notebook.createNote();
AuthenticationInfo subject = new AuthenticationInfo(message.principal);
Note note = notebook.createNote(subject);
note.addParagraph(); // it's an empty note. so add one paragraph
if (message != null) {
String noteName = (String) message.get("name");
@ -578,10 +588,10 @@ public class NotebookServer extends WebSocketServlet implements
note.setName(noteName);
}
note.persist();
note.persist(subject);
addConnectionToNote(note.id(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
broadcastNoteList();
broadcastNoteList(subject);
}
private void removeNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -595,13 +605,15 @@ public class NotebookServer extends WebSocketServlet implements
Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isOwner(noteId, userAndRoles)) {
permissionError(conn, "remove", userAndRoles, notebookAuthorization.getOwners(noteId));
permissionError(conn, "remove", fromMessage.principal,
userAndRoles, notebookAuthorization.getOwners(noteId));
return;
}
notebook.removeNote(noteId);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook.removeNote(noteId, subject);
removeNote(noteId);
broadcastNoteList();
broadcastNoteList(subject);
}
private void updateParagraph(NotebookSocket conn, HashSet<String> userAndRoles,
@ -618,8 +630,10 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
@ -628,7 +642,7 @@ public class NotebookServer extends WebSocketServlet implements
p.setConfig(config);
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
note.persist();
note.persist(subject);
broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
@ -637,10 +651,11 @@ public class NotebookServer extends WebSocketServlet implements
throws IOException, CloneNotSupportedException {
String noteId = getOpenNoteId(conn);
String name = (String) fromMessage.get("name");
Note newNote = notebook.cloneNote(noteId, name);
Note newNote = notebook.cloneNote(noteId, name, new AuthenticationInfo(fromMessage.principal));
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.id(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
broadcastNoteList();
broadcastNoteList(subject);
}
protected Note importNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -650,10 +665,11 @@ public class NotebookServer extends WebSocketServlet implements
if (fromMessage != null) {
String noteName = (String) ((Map) fromMessage.get("notebook")).get("name");
String noteJson = gson.toJson(fromMessage.get("notebook"));
note = notebook.importNote(noteJson, noteName);
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note = notebook.importNote(noteJson, noteName, subject);
note.persist(subject);
broadcastNote(note);
broadcastNoteList();
broadcastNoteList(subject);
}
return note;
}
@ -667,15 +683,17 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
/** We dont want to remove the last paragraph */
if (!note.isLastParagraph(paragraphId)) {
note.removeParagraph(paragraphId);
note.persist();
note.persist(subject);
broadcastNote(note);
}
}
@ -690,7 +708,8 @@ public class NotebookServer extends WebSocketServlet implements
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
@ -964,13 +983,15 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
note.moveParagraph(paragraphId, newIndex);
note.persist();
note.persist(subject);
broadcastNote(note);
}
@ -981,13 +1002,15 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
note.insertParagraph(index);
note.persist();
note.persist(subject);
broadcastNote(note);
}
@ -1002,7 +1025,8 @@ public class NotebookServer extends WebSocketServlet implements
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
@ -1021,7 +1045,8 @@ public class NotebookServer extends WebSocketServlet implements
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId));
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
@ -1047,11 +1072,13 @@ public class NotebookServer extends WebSocketServlet implements
// if it's the last paragraph, let's add a new one
boolean isTheLastParagraph = note.getLastParagraph().getId()
.equals(p.getId());
note.setLastReplName(paragraphId);
if (!Strings.isNullOrEmpty(text) && isTheLastParagraph) {
note.addParagraph();
}
note.persist();
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note.persist(subject);
try {
note.run(paragraphId);
} catch (Exception ex) {
@ -1090,7 +1117,8 @@ public class NotebookServer extends WebSocketServlet implements
Message fromMessage) throws IOException {
String noteId = (String) fromMessage.get("noteId");
String commitMessage = (String) fromMessage.get("commitMessage");
notebook.checkpointNote(noteId, commitMessage);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook.checkpointNote(noteId, commitMessage, subject);
}
/**
@ -1213,7 +1241,8 @@ public class NotebookServer extends WebSocketServlet implements
if (job.isTerminated()) {
LOG.info("Job {} is finished", job.getId());
try {
note.persist();
//TODO(khalid): may change interface for JobListener and pass subject from interpreter
note.persist(null);
} catch (IOException e) {
LOG.error(e.toString(), e);
}

57
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -16,24 +16,35 @@
*/
package org.apache.zeppelin.utils;
import org.apache.shiro.realm.Realm;
import org.apache.shiro.realm.text.IniRealm;
import org.apache.shiro.subject.Subject;
import org.apache.shiro.util.ThreadContext;
import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.shiro.mgt.SecurityManager;
import org.apache.shiro.config.IniSecurityManagerFactory;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.*;
/**
* Tools for securing Zeppelin
*/
public class SecurityUtils {
public static void initSecurityManager(String shiroPath) {
IniSecurityManagerFactory factory = new IniSecurityManagerFactory("file:" + shiroPath);
SecurityManager securityManager = factory.getInstance();
org.apache.shiro.SecurityUtils.setSecurityManager(securityManager);
}
public static Boolean isValidOrigin(String sourceHost, ZeppelinConfiguration conf)
throws UnknownHostException, URISyntaxException {
if (sourceHost == null || sourceHost.isEmpty()){
if (sourceHost == null || sourceHost.isEmpty()) {
return false;
}
String sourceUriHost = new URI(sourceHost).getHost();
@ -43,13 +54,14 @@ public class SecurityUtils {
String currentHost = InetAddress.getLocalHost().getHostName().toLowerCase();
return conf.getAllowedOrigins().contains("*") ||
currentHost.equals(sourceUriHost) ||
"localhost".equals(sourceUriHost) ||
conf.getAllowedOrigins().contains(sourceHost);
currentHost.equals(sourceUriHost) ||
"localhost".equals(sourceUriHost) ||
conf.getAllowedOrigins().contains(sourceHost);
}
/**
* Return the authenticated user if any otherwise returns "anonymous"
*
* @return shiro principal
*/
public static String getPrincipal() {
@ -58,26 +70,49 @@ public class SecurityUtils {
String principal;
if (subject.isAuthenticated()) {
principal = subject.getPrincipal().toString();
}
else {
} else {
principal = "anonymous";
}
return principal;
}
public static Collection getRealmsList() {
DefaultWebSecurityManager defaultWebSecurityManager;
String key = ThreadContext.SECURITY_MANAGER_KEY;
defaultWebSecurityManager = (DefaultWebSecurityManager) ThreadContext.get(key);
Collection<Realm> realms = defaultWebSecurityManager.getRealms();
return realms;
}
/**
* Return the roles associated with the authenticated user if any otherwise returns empty set
* TODO(prasadwagle) Find correct way to get user roles (see SHIRO-492)
*
* @return shiro roles
*/
public static HashSet<String> getRoles() {
Subject subject = org.apache.shiro.SecurityUtils.getSubject();
HashSet<String> roles = new HashSet<>();
Map allRoles = null;
if (subject.isAuthenticated()) {
for (String role : Arrays.asList("role1", "role2", "role3")) {
if (subject.hasRole(role)) {
roles.add(role);
Collection realmsList = SecurityUtils.getRealmsList();
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getName();
if (name.equals("iniRealm")) {
allRoles = ((IniRealm) realm).getIni().get("roles");
break;
}
}
if (allRoles != null) {
Iterator it = allRoles.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
if (subject.hasRole((String) pair.getKey())) {
roles.add((String) pair.getKey());
}
}
}
}

12
zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
View file

@ -22,6 +22,9 @@ import com.google.common.base.Function;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.openqa.selenium.*;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.FluentWait;
import org.openqa.selenium.support.ui.Wait;
@ -30,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@ -38,7 +42,7 @@ import static org.openqa.selenium.Keys.ENTER;
import static org.openqa.selenium.Keys.SHIFT;
abstract public class AbstractZeppelinIT {
protected WebDriver driver;
protected static WebDriver driver;
protected final static Logger LOG = LoggerFactory.getLogger(AbstractZeppelinIT.class);
protected static final long MAX_IMPLICIT_WAIT = 30;
@ -114,7 +118,7 @@ abstract public class AbstractZeppelinIT {
});
}
protected boolean endToEndTestEnabled() {
protected static boolean endToEndTestEnabled() {
return null != System.getenv("TEST_SELENIUM");
}
@ -151,6 +155,10 @@ abstract public class AbstractZeppelinIT {
protected void handleException(String message, Exception e) throws Exception {
LOG.error(message, e);
LogEntries logEntries = driver.manage().logs().get(LogType.BROWSER);
for (LogEntry entry : logEntries) {
LOG.error(new Date(entry.getTimestamp()) + " " + entry.getLevel() + " " + entry.getMessage());
}
File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
LOG.error("ScreenShot::\ndata:image/png;base64," + new String(Base64.encodeBase64(FileUtils.readFileToByteArray(scrFile))));
throw e;

3
zeppelin-server/src/test/java/org/apache/zeppelin/WebDriverManager.java
View file

@ -125,7 +125,8 @@ public class WebDriverManager {
(new WebDriverWait(driver, 5)).until(new ExpectedCondition<Boolean>() {
@Override
public Boolean apply(WebDriver d) {
return d.findElement(By.partialLinkText("Create new note"))
return d.findElement(By.xpath(
"//div[contains(@class, 'navbar-collapse')]//li//a[contains(.,'Connected')]"))
.isDisplayed();
}
});

7
zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinITUtils.java
View file

@ -39,4 +39,11 @@ public class ZeppelinITUtils {
LOG.info("Finished.");
}
}
public static void restartZeppelin() {
CommandExecutor.executeCommandLocalHost("../bin/zeppelin-daemon.sh restart",
false, ProcessData.Types_Of_Data.OUTPUT);
//wait for server to start.
sleep(5000, false);
}
}

209
zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java Normal file
View file

@ -0,0 +1,209 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.integration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.apache.zeppelin.ZeppelinITUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.Keys;
import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.List;
/**
* Created for org.apache.zeppelin.integration on 13/06/16.
*/
public class AuthenticationIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(AuthenticationIT.class);
@Rule
public ErrorCollector collector = new ErrorCollector();
static String authShiro = "[users]\n" +
"admin = password1, admin\n" +
"finance1 = finance1, finance\n" +
"finance2 = finance2, finance\n" +
"hr1 = hr1, hr\n" +
"hr2 = hr2, hr\n" +
"[main]\n" +
"sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" +
"securityManager.sessionManager = $sessionManager\n" +
"securityManager.sessionManager.globalSessionTimeout = 86400000\n" +
"shiro.loginUrl = /api/login\n" +
"[roles]\n" +
"admin = *\n" +
"hr = *\n" +
"finance = *\n" +
"[urls]\n" +
"/api/version = anon\n" +
"/** = authc";
static String originalShiro = "";
@BeforeClass
public static void startUp() {
if (!endToEndTestEnabled()) {
return;
}
try {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
File file = new File(conf.getShiroPath());
originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
FileUtils.write(file, authShiro, "UTF-8");
} catch (IOException e) {
LOG.error("Error in AuthenticationIT startUp::", e);
}
ZeppelinITUtils.restartZeppelin();
driver = WebDriverManager.getWebDriver();
}
@AfterClass
public static void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
try {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
File file = new File(conf.getShiroPath());
FileUtils.write(file, originalShiro, "UTF-8");
} catch (IOException e) {
LOG.error("Error in AuthenticationIT tearDown::", e);
}
ZeppelinITUtils.restartZeppelin();
driver.quit();
}
private void authenticationUser(String userName, String password) {
pollingWait(By.xpath(
"//div[contains(@class, 'navbar-collapse')]//li//button[contains(.,'Login')]"),
MAX_BROWSER_TIMEOUT_SEC).click();
sleep(1000, false);
pollingWait(By.xpath("//*[@id='userName']"), MAX_BROWSER_TIMEOUT_SEC).sendKeys(userName);
pollingWait(By.xpath("//*[@id='password']"), MAX_BROWSER_TIMEOUT_SEC).sendKeys(password);
pollingWait(By.xpath("//*[@id='NoteImportCtrl']//button[contains(.,'Login')]"),
MAX_BROWSER_TIMEOUT_SEC).click();
sleep(1000, false);
}
private void logoutUser(String userName) {
sleep(500, false);
driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
userName + "')]")).click();
sleep(500, false);
driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
userName + "')]//a[@ng-click='logout()']")).click();
sleep(5000, false);
}
// @Test
public void testSimpleAuthentication() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
AuthenticationIT authenticationIT = new AuthenticationIT();
authenticationIT.authenticationUser("admin", "password1");
collector.checkThat("Check is user logged in", true,
CoreMatchers.equalTo(driver.findElement(By.partialLinkText("Create new note"))
.isDisplayed()));
authenticationIT.logoutUser("admin");
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testCreateNewButton ", e);
}
}
@Test
public void testGroupPermission() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
AuthenticationIT authenticationIT = new AuthenticationIT();
authenticationIT.authenticationUser("finance1", "finance1");
createNewNote();
String noteId = driver.getCurrentUrl().substring(driver.getCurrentUrl().lastIndexOf("/") + 1);
pollingWait(By.xpath("//button[@tooltip='Note permissions']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys(Keys.ENTER);
pollingWait(By.xpath("//input[@ng-model='permissions.owners']"), MAX_BROWSER_TIMEOUT_SEC)
.sendKeys("finance");
pollingWait(By.xpath("//input[@ng-model='permissions.readers']"), MAX_BROWSER_TIMEOUT_SEC)
.sendKeys("finance");
pollingWait(By.xpath("//input[@ng-model='permissions.writers']"), MAX_BROWSER_TIMEOUT_SEC)
.sendKeys("finance");
pollingWait(By.xpath("//button[@ng-click='savePermissions()']"), MAX_BROWSER_TIMEOUT_SEC)
.sendKeys(Keys.ENTER);
pollingWait(By.xpath("//div[@class='modal-dialog'][contains(.,'Permissions Saved ')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]"),
MAX_BROWSER_TIMEOUT_SEC).click();
authenticationIT.logoutUser("finance1");
authenticationIT.authenticationUser("hr1", "hr1");
pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
MAX_BROWSER_TIMEOUT_SEC).click();
List<WebElement> privilegesModal = driver.findElements(
By.xpath("//div[@class='modal-content']//div[@class='bootstrap-dialog-header']" +
"//div[contains(.,'Insufficient privileges')]"));
collector.checkThat("Check is user has permission to view this notebook", 1,
CoreMatchers.equalTo(privilegesModal.size()));
driver.findElement(
By.xpath("//div[@class='modal-content'][contains(.,'Insufficient privileges')]" +
"//div[@class='modal-footer']//button[2]")).click();
authenticationIT.logoutUser("hr1");
authenticationIT.authenticationUser("finance2", "finance2");
pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
MAX_BROWSER_TIMEOUT_SEC).click();
privilegesModal = driver.findElements(
By.xpath("//div[@class='modal-content']//div[@class='bootstrap-dialog-header']" +
"//div[contains(.,'Insufficient privileges')]"));
collector.checkThat("Check is user has permission to view this notebook", 0,
CoreMatchers.equalTo(privilegesModal.size()));
deleteTestNotebook(driver);
authenticationIT.logoutUser("finance2");
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testGroupPermission ", e);
}
}
}

15
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.integration;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.apache.zeppelin.ZeppelinITUtils;
@ -28,14 +29,11 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.*;
import org.openqa.selenium.interactions.Action;
import org.openqa.selenium.interactions.Actions;
import org.openqa.selenium.support.ui.Select;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
public class ParagraphActionsIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(ParagraphActionsIT.class);
@ -94,21 +92,28 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
ZeppelinITUtils.sleep(1000, false);
waitForParagraph(1, "READY");
String oldIntpTag = driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created above",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.equalTo(""));
CoreMatchers.not(StringUtils.EMPTY));
setTextOfParagraph(1, " this is above ");
newPara = driver.findElement(By.xpath(getParagraphXPath(2) + "//div[contains(@class,'new-paragraph')][2]"));
action.moveToElement(newPara).click().build().perform();
waitForParagraph(3, "READY");
String lastIntpTag = driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText();
collector.checkThat("Paragraph is created below",
driver.findElement(By.xpath(getParagraphXPath(3) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.equalTo(""));
CoreMatchers.not(StringUtils.EMPTY));
setTextOfParagraph(3, " this is below ");
collector.checkThat("Compare interpreter name tag", oldIntpTag, CoreMatchers.equalTo(lastIntpTag));
collector.checkThat("The output field of paragraph1 contains",
driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'editor')]")).getText(),
CoreMatchers.equalTo(" this is above "));

8
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -123,7 +123,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testInterpreterAutoBinding() throws IOException {
// create note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
@ -136,13 +136,13 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
@ -179,7 +179,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
}
assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
//cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test

8
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -61,7 +61,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
@Test
public void testPermissions() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote();
Note note1 = ZeppelinServer.notebook.createNote(null);
// Set only readers
String jsonRequest = "{\"readers\":[\"admin-team\"],\"owners\":[]," +
"\"writers\":[]}";
@ -84,7 +84,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
get.releaseConnection();
Note note2 = ZeppelinServer.notebook.createNote();
Note note2 = ZeppelinServer.notebook.createNote(null);
// Set only writers
jsonRequest = "{\"readers\":[],\"owners\":[]," +
"\"writers\":[\"admin-team\"]}";
@ -118,8 +118,8 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
assertEquals(authInfo.get("owners"), Lists.newArrayList());
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId());
ZeppelinServer.notebook.removeNote(note2.getId());
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(note2.getId(), null);
}
}

97
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
View file

@ -77,7 +77,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testGetNotebookInfo() throws IOException {
LOG.info("testGetNotebookInfo");
// Create note to get info
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("note");
Paragraph paragraph = note.addParagraph();
@ -86,7 +86,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
String paragraphText = "%md This is my new paragraph in my new note";
paragraph.setText(paragraphText);
note.persist();
note.persist(null);
String sourceNoteID = note.getId();
GetMethod get = httpGet("/notebook/" + sourceNoteID);
@ -146,14 +146,15 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("compare note name", expectedNoteName, newNoteName);
assertEquals("initial paragraph check failed", 3, newNote.getParagraphs().size());
for (Paragraph p : newNote.getParagraphs()) {
if (StringUtils.isEmpty(p.getText())) {
if (StringUtils.isEmpty(p.getText()) ||
p.getText().trim().equals(newNote.getLastInterpreterName())) {
continue;
}
assertTrue("paragraph title check failed", p.getTitle().startsWith("title"));
assertTrue("paragraph text check failed", p.getText().startsWith("text"));
}
// cleanup
ZeppelinServer.notebook.removeNote(newNotebookId);
ZeppelinServer.notebook.removeNote(newNotebookId, null);
post.releaseConnection();
}
@ -180,7 +181,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
assertEquals("compare note name", expectedNoteName, newNoteName);
// cleanup
ZeppelinServer.notebook.removeNote(newNotebookId);
ZeppelinServer.notebook.removeNote(newNotebookId, null);
post.releaseConnection();
}
@ -189,7 +190,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testDeleteNote() throws IOException {
LOG.info("testDeleteNote");
//Create note and get ID
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
String noteId = note.getId();
testDeleteNotebook(noteId);
}
@ -205,7 +206,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
@Test
public void testExportNotebook() throws IOException {
LOG.info("testExportNotebook");
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("source note for export");
Paragraph paragraph = note.addParagraph();
@ -213,7 +214,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist();
note.persist(null);
String sourceNoteID = note.getId();
// Call export Notebook REST API
GetMethod get = httpGet("/notebook/export/" + sourceNoteID);
@ -227,7 +228,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String exportJSON = (String) resp.get("body");
assertNotNull("Can not find new notejson", exportJSON);
LOG.info("export JSON:=" + exportJSON);
ZeppelinServer.notebook.removeNote(sourceNoteID);
ZeppelinServer.notebook.removeNote(sourceNoteID, null);
get.releaseConnection();
}
@ -238,7 +239,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String noteName = "source note for import";
LOG.info("testImortNotebook");
// create test notebook
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName(noteName);
Paragraph paragraph = note.addParagraph();
@ -246,7 +247,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist();
note.persist(null);
String sourceNoteID = note.getId();
// get note content as JSON
String oldJson = getNoteContent(sourceNoteID);
@ -264,8 +265,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs()
.size());
// cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(newNote.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(newNote.getId(), null);
importPost.releaseConnection();
}
@ -300,7 +301,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testCloneNotebook() throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("testCloneNotebook");
// Create note to clone
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("source note for clone");
Paragraph paragraph = note.addParagraph();
@ -308,7 +309,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist();
note.persist(null);
String sourceNoteID = note.getId();
String noteName = "clone Note Name";
@ -328,8 +329,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("Compare note names", noteName, newNote.getName());
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs().size());
//cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(newNote.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(newNote.getId(), null);
post.releaseConnection();
}
@ -349,7 +350,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testNoteJobs() throws IOException, InterruptedException {
LOG.info("testNoteJobs");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -359,7 +360,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%md This is test paragraph.");
note.persist();
note.persist(null);
String noteID = note.getId();
note.runAll();
@ -397,14 +398,14 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
Thread.sleep(1000);
//cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testGetNotebookJob() throws IOException, InterruptedException {
LOG.info("testGetNotebookJob");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -414,7 +415,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%sh sleep 1");
note.persist();
note.persist(null);
String noteID = note.getId();
note.runAll();
@ -450,14 +451,14 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
}
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testRunParagraphWithParams() throws IOException, InterruptedException {
LOG.info("testRunParagraphWithParams");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -467,7 +468,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
note.persist();
note.persist(null);
String noteID = note.getId();
note.runAll();
@ -495,13 +496,13 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("world", params.get("param2"));
//cleanup
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testCronJobs() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -545,18 +546,18 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
DeleteMethod deleteCron = httpDelete("/notebook/cron/" + note.getId());
assertThat("", deleteCron, isAllowed());
deleteCron.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testRegressionZEPPELIN_527() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
note.persist();
note.persist(null);
GetMethod getNoteJobs = httpGet("/notebook/job/" + note.getId());
assertThat("test notebook jobs run:", getNoteJobs, isAllowed());
@ -567,12 +568,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertFalse(body.get(0).containsKey("finished"));
getNoteJobs.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testInsertParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"text1\"}";
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
@ -607,17 +608,17 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("title2", paragraphAtIdx0.getTitle());
assertEquals("text2", paragraphAtIdx0.getText());
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testGetParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
Paragraph p = note.addParagraph();
p.setTitle("hello");
p.setText("world");
note.persist();
note.persist(null);
GetMethod get = httpGet("/notebook/" + note.getId() + "/paragraph/" + p.getId());
LOG.info("testGetParagraph response\n" + get.getResponseBodyAsString());
@ -636,12 +637,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("hello", body.get("title"));
assertEquals("world", body.get("text"));
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testMoveParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
Paragraph p = note.addParagraph();
p.setTitle("title1");
@ -651,7 +652,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
p2.setTitle("title2");
p2.setText("text2");
note.persist();
note.persist(null);
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph/" + p2.getId() + "/move/" + 0, "");
assertThat("Test post method: ", post, isAllowed());
@ -668,18 +669,18 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertThat("Test post method: ", post2, isBadRequest());
post.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
public void testDeleteParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
Paragraph p = note.addParagraph();
p.setTitle("title1");
p.setText("text1");
note.persist();
note.persist(null);
DeleteMethod delete = httpDelete("/notebook/" + note.getId() + "/paragraph/" + p.getId());
assertThat("Test delete method: ", delete, isAllowed());
@ -689,7 +690,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
Paragraph retrParagrah = retrNote.getParagraph(p.getId());
assertNull("paragraph should be deleted", retrParagrah);
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
@Test
@ -705,12 +706,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String username = body.get("principal");
getSecurityTicket.releaseConnection();
Note note1 = ZeppelinServer.notebook.createNote();
Note note1 = ZeppelinServer.notebook.createNote(null);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 1\"}";
PostMethod postNotebookText = httpPost("/notebook/" + note1.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
Note note2 = ZeppelinServer.notebook.createNote();
Note note2 = ZeppelinServer.notebook.createNote(null);
jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 2\"}";
postNotebookText = httpPost("/notebook/" + note2.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
@ -752,13 +753,13 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
getPermission.releaseConnection();
}
searchNotebook.releaseConnection();
ZeppelinServer.notebook.removeNote(note1.getId());
ZeppelinServer.notebook.removeNote(note2.getId());
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(note2.getId(), null);
}
@Test
public void testTitleSearch() throws IOException {
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
String jsonRequest = "{\"title\": \"testTitleSearchOfParagraph\", \"text\": \"ThisIsToTestSearchMethodWithTitle \"}";
PostMethod postNotebookText = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
@ -779,7 +780,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
assertEquals("Paragraph title hits must be at-least one", true, numberOfTitleHits >= 1);
searchNotebook.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId());
ZeppelinServer.notebook.removeNote(note.getId(), null);
}
}

22
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -73,7 +73,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
@Test
public void basicRDDTransformationAndActionTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
// run markdown paragraph, again
Paragraph p = note.addParagraph();
@ -85,13 +85,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55", p.getResult().message());
ZeppelinServer.notebook.removeNote(note.id());
ZeppelinServer.notebook.removeNote(note.id(), null);
}
@Test
public void sparkRTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
int sparkVersion = getSparkVersionNumber(note);
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
@ -111,7 +111,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[1] 3", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id());
ZeppelinServer.notebook.removeNote(note.id(), null);
}
@Test
@ -138,7 +138,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
@Test
public void pySparkTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note");
int sparkVersion = getSparkVersionNumber(note);
@ -155,13 +155,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id());
ZeppelinServer.notebook.removeNote(note.id(), null);
}
@Test
public void pySparkAutoConvertOptionTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note");
int sparkVersion = getSparkVersionNumber(note);
@ -180,13 +180,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("10\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.id());
ZeppelinServer.notebook.removeNote(note.id(), null);
}
@Test
public void zRunTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
Paragraph p0 = note.addParagraph();
Map config0 = p0.getConfig();
config0.put("enabled", true);
@ -212,13 +212,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p2.getStatus());
assertEquals("10", p2.getResult().message());
ZeppelinServer.notebook.removeNote(note.id());
ZeppelinServer.notebook.removeNote(note.id(), null);
}
@Test
public void pySparkDepLoaderTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote();
Note note = ZeppelinServer.notebook.createNote(null);
if (isPyspark() && getSparkVersionNumber(note) >= 14) {
// restart spark interpreter

6
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -89,7 +89,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
@Test
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
// create a notebook
Note note1 = notebook.createNote();
Note note1 = notebook.createNote(null);
// get reference to interpreterGroup
InterpreterGroup interpreterGroup = null;
@ -139,7 +139,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
verify(sock1, times(0)).send(anyString());
verify(sock2, times(1)).send(anyString());
notebook.removeNote(note1.getId());
notebook.removeNote(note1.getId(), null);
}
@Test
@ -162,7 +162,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
assertNotEquals(null, notebook.getNote(note.getId()));
assertEquals("Test Zeppelin notebook import", notebook.getNote(note.getId()).getName());
assertEquals("Test paragraphs import", notebook.getNote(note.getId()).getParagraphs().get(0).getText());
notebook.removeNote(note.getId());
notebook.removeNote(note.getId(), null);
}
@Test

3
zeppelin-web/bower.json
View file

@ -36,9 +36,6 @@
"angular-mocks": "1.5.0"
},
"appPath": "src",
"resolutions": {
"perfect-scrollbar": "~0.5.4"
},
"overrides": {
"ace-builds": {
"main": [

1
zeppelin-web/package.json
View file

@ -1,5 +1,6 @@
{
"name": "zeppelin-web",
"license": "Apache-2.0",
"version": "0.0.0",
"dependencies": {
"grunt-angular-templates": "^0.5.7",

2
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -34,7 +34,7 @@ limitations under the License.
</div>
<div class="row">
<div class="col-md-12">
Manage interpreters settings. You can create create / remove settings. Note can bind/unbind these interpreter settings.
Manage interpreters settings. You can create / edit / remove settings. Note can bind / unbind these interpreter settings.
</div>
</div>

21
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -151,21 +151,26 @@ limitations under the License.
</span>
<div class="pull-right" style="margin-top:15px; margin-right:15px; font-size:15px;">
<span style="position:relative; top:3px; margin-right:4px; cursor:pointer"
<span>
<button type="button"
class="btn btn-default btn-xs"
data-toggle="modal"
data-target="#shortcutModal"
tooltip-placement="bottom" tooltip="List of shortcut">
<i class="icon-question"></i>
</span>
<span style="position:relative; top:2px; margin-right:4px; cursor:pointer;"
<i class="icon-question"></i>
</button>
<button type="button"
class="btn btn-default btn-xs"
ng-click="toggleSetting()"
tooltip-placement="bottom" tooltip="Interpreter binding">
<i class="fa fa-cog" ng-style="{color: showSetting ? '#3071A9' : 'black' }"></i>
</span>
<span style="position:relative; top:2px; margin-right:4px; cursor:pointer;"
<i class="fa fa-cog" ng-style="{color: showSetting ? '#3071A9' : 'black' }"></i>
</button>
<button type="button"
class="btn btn-default btn-xs"
ng-click="togglePermissions()"
tooltip-placement="bottom" tooltip="Note permissions">
<i class="fa fa-lock" ng-style="{color: showPermissions ? '#3071A9' : 'black' }"></i>
<i class="fa fa-lock" ng-style="{color: showPermissions ? '#3071A9' : 'black' }"></i>
</button>
</span>
<span class="btn-group">

4
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -879,6 +879,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
// function to find suggestion list on change
$scope.search = function(role) {
angular.element('.userlist').show();
convertToArray(role);
checkPreviousRole(role);
getChangedIndex();
@ -950,5 +951,8 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
$scope.suggestions = [];
};
angular.element(document).click(function(){
angular.element('.userlist').hide();
});
});

25
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -490,7 +490,7 @@ angular.module('zeppelinWebApp')
if (statusChanged || resultRefreshed) {
// when last paragraph runs, zeppelin automatically appends new paragraph.
// this broadcast will focus to the newly inserted paragraph
var paragraphs = angular.element('div[id$="_paragraphColumn_main"');
var paragraphs = angular.element('div[id$="_paragraphColumn_main"]');
if (paragraphs.length >= 2 && paragraphs[paragraphs.length-2].id.startsWith($scope.paragraph.id)) {
// rendering output can took some time. So delay scrolling event firing for sometime.
setTimeout(function() {
@ -498,7 +498,6 @@ angular.module('zeppelinWebApp')
}, 500);
}
}
}
});
@ -578,7 +577,7 @@ angular.module('zeppelinWebApp')
};
$scope.removeParagraph = function() {
var paragraphs = angular.element('div[id$="_paragraphColumn_main"');
var paragraphs = angular.element('div[id$="_paragraphColumn_main"]');
if (paragraphs[paragraphs.length-1].id.startsWith($scope.paragraph.id)) {
BootstrapDialog.alert({
closable: true,
@ -787,6 +786,7 @@ angular.module('zeppelinWebApp')
$scope.editor.setTheme('ace/theme/chrome');
if ($scope.paragraphFocused) {
$scope.editor.focus();
$scope.goToLineEnd();
}
autoAdjustEditorHeight(_editor.container.id);
@ -960,7 +960,7 @@ angular.module('zeppelinWebApp')
$rootScope.$on('scrollToCursor', function(event) {
// scroll on 'scrollToCursor' event only when cursor is in the last paragraph
var paragraphs = angular.element('div[id$="_paragraphColumn_main"');
var paragraphs = angular.element('div[id$="_paragraphColumn_main"]');
if (paragraphs[paragraphs.length-1].id.startsWith($scope.paragraph.id)) {
$scope.scrollToCursor($scope.paragraph.id, 0);
}
@ -1056,6 +1056,10 @@ angular.module('zeppelinWebApp')
return false;
};
$scope.goToLineEnd = function () {
$scope.editor.navigateLineEnd();
};
$scope.$on('updateProgress', function(event, data) {
if (data.id === $scope.paragraph.id) {
$scope.currentProgress = data.progress;
@ -1108,19 +1112,6 @@ angular.module('zeppelinWebApp')
} else if (keyEvent.ctrlKey && keyEvent.shiftKey && keyCode === 187) { // Ctrl + Shift + =
$scope.paragraph.config.colWidth = Math.min(12, $scope.paragraph.config.colWidth + 1);
$scope.changeColWidth();
} else if (keyEvent.ctrlKey && keyEvent.altKey && ((keyCode >= 48 && keyCode <=57) || keyCode === 189 || keyCode === 187)) { // Ctrl + Alt + [1~9,0,-,=]
var colWidth = 12;
if (keyCode === 48) {
colWidth = 10;
} else if (keyCode === 189) {
colWidth = 11;
} else if (keyCode === 187) {
colWidth = 12;
} else {
colWidth = keyCode - 48;
}
$scope.paragraph.config.colWidth = colWidth;
$scope.changeColWidth();
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 84) { // Ctrl + Alt + t
if ($scope.paragraph.config.title) {
$scope.hideTitle();

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -501,7 +501,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ "org.apache.zeppelin.file.HDFSFileInterpreter,"
+ "org.apache.zeppelin.phoenix.PhoenixInterpreter,"
+ "org.apache.zeppelin.postgresql.PostgreSqlInterpreter,"
+ "org.apache.zeppelin.flink.FlinkInterpreter,"
+ "org.apache.zeppelin.python.PythonInterpreter,"

87
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -20,11 +20,12 @@ package org.apache.zeppelin.notebook;
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
@ -41,6 +42,7 @@ import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.search.SearchService;
import com.google.common.base.Optional;
import com.google.gson.Gson;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
@ -66,6 +68,7 @@ public class Note implements Serializable, ParagraphJobListener {
private String name = "";
private String id;
private AtomicReference<String> lastReplName = new AtomicReference<>(StringUtils.EMPTY);
private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create();
@SuppressWarnings("rawtypes")
@ -112,6 +115,17 @@ public class Note implements Serializable, ParagraphJobListener {
id = IdHashes.encode(System.currentTimeMillis() + new Random().nextInt());
}
private String getDefaultInterpreterName() {
Optional<InterpreterSetting> settingOptional = replLoader.getDefaultInterpreterSetting();
return settingOptional.isPresent() ? settingOptional.get().getName() : StringUtils.EMPTY;
}
void putDefaultReplName() {
String defaultInterpreterName = getDefaultInterpreterName();
logger.info("defaultInterpreterName is '{}'", defaultInterpreterName);
lastReplName.set(defaultInterpreterName);
}
public String id() {
return id;
}
@ -192,6 +206,7 @@ public class Note implements Serializable, ParagraphJobListener {
public Paragraph addParagraph() {
Paragraph p = new Paragraph(this, this, replLoader);
addLastReplNameIfEmptyText(p);
synchronized (paragraphs) {
paragraphs.add(p);
}
@ -241,6 +256,7 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public Paragraph insertParagraph(int index) {
Paragraph p = new Paragraph(this, this, replLoader);
addLastReplNameIfEmptyText(p);
synchronized (paragraphs) {
paragraphs.add(index, p);
}
@ -250,6 +266,22 @@ public class Note implements Serializable, ParagraphJobListener {
return p;
}
/**
* Add Last Repl name If Paragraph has empty text
*
* @param p Paragraph
*/
private void addLastReplNameIfEmptyText(Paragraph p) {
String replName = lastReplName.get();
if (StringUtils.isEmpty(p.getText()) && StringUtils.isNotEmpty(replName)) {
p.setText(getInterpreterName(replName) + " ");
}
}
private String getInterpreterName(String replName) {
return StringUtils.isBlank(replName) ? StringUtils.EMPTY : "%" + replName;
}
/**
* Remove paragraph by id.
*
@ -402,6 +434,9 @@ public class Note implements Serializable, ParagraphJobListener {
public void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
synchronized (paragraphs) {
if (!paragraphs.isEmpty()) {
setLastReplName(paragraphs.get(paragraphs.size() - 1));
}
for (Paragraph p : paragraphs) {
if (!p.isEnabled()) {
continue;
@ -445,6 +480,22 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
/**
* Check whether all paragraphs belongs to this note has terminated
* @return
*/
public boolean isTerminated() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (!p.isTerminated()) {
return false;
}
}
}
return true;
}
public List<InterpreterCompletion> completion(String paragraphId, String buffer, int cursor) {
Paragraph p = getParagraph(paragraphId);
p.setNoteReplLoader(replLoader);
@ -513,27 +564,37 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
public void persist() throws IOException {
public void persist(AuthenticationInfo subject) throws IOException {
stopDelayedPersistTimer();
snapshotAngularObjectRegistry();
index.updateIndexDoc(this);
repo.save(this);
repo.save(this, subject);
}
private void setLastReplName(Paragraph lastParagraphStarted) {
if (StringUtils.isNotEmpty(lastParagraphStarted.getRequiredReplName())) {
lastReplName.set(lastParagraphStarted.getRequiredReplName());
}
}
public void setLastReplName(String paragraphId) {
setLastReplName(getParagraph(paragraphId));
}
/**
* Persist this note with maximum delay.
* @param maxDelaySec
*/
public void persist(int maxDelaySec) {
startDelayedPersistTimer(maxDelaySec);
public void persist(int maxDelaySec, AuthenticationInfo subject) {
startDelayedPersistTimer(maxDelaySec, subject);
}
public void unpersist() throws IOException {
repo.remove(id());
public void unpersist(AuthenticationInfo subject) throws IOException {
repo.remove(id(), subject);
}
private void startDelayedPersistTimer(int maxDelaySec) {
private void startDelayedPersistTimer(int maxDelaySec, final AuthenticationInfo subject) {
synchronized (this) {
if (delayedPersist != null) {
return;
@ -544,7 +605,7 @@ public class Note implements Serializable, ParagraphJobListener {
@Override
public void run() {
try {
persist();
persist(subject);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
@ -585,6 +646,14 @@ public class Note implements Serializable, ParagraphJobListener {
this.info = info;
}
public String getLastReplName() {
return lastReplName.get();
}
public String getLastInterpreterName() {
return getInterpreterName(getLastReplName());
}
@Override
public void beforeStatusChange(Job job, Status before, Status after) {
if (jobListenerFactory != null) {

18
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java
View file

@ -17,6 +17,8 @@
package org.apache.zeppelin.notebook;
import com.google.common.base.Optional;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
@ -120,7 +122,7 @@ public class NoteInterpreterLoader {
if (replName == null || replName.trim().length() == 0) {
// get default settings (first available)
InterpreterSetting defaultSettings = settings.get(0);
InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings).get();
return createOrGetInterpreterList(defaultSettings).get(0);
}
@ -157,7 +159,7 @@ public class NoteInterpreterLoader {
} else {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
InterpreterSetting defaultSetting = settings.get(0);
InterpreterSetting defaultSetting = getDefaultInterpreterSetting(settings).get();
Interpreter.RegisteredInterpreter registeredInterpreter =
Interpreter.registeredInterpreters.get(defaultSetting.getGroup() + "." + replName);
if (registeredInterpreter != null) {
@ -196,4 +198,16 @@ public class NoteInterpreterLoader {
return null;
}
private Optional<InterpreterSetting>
getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
if (settings == null || settings.isEmpty()) {
return Optional.absent();
}
return Optional.of(settings.get(0));
}
Optional<InterpreterSetting> getDefaultInterpreterSetting() {
return getDefaultInterpreterSetting(getInterpreterSettings());
}
}

55
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -44,6 +44,7 @@ import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
@ -134,12 +135,12 @@ public class Notebook implements NoteEventListener {
* @return
* @throws IOException
*/
public Note createNote() throws IOException {
public Note createNote(AuthenticationInfo subject) throws IOException {
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
note = createNote(replFactory.getDefaultInterpreterSettingList());
note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
} else {
note = createNote(null);
note = createNote(null, subject);
}
notebookIndex.addIndexDoc(note);
return note;
@ -151,7 +152,8 @@ public class Notebook implements NoteEventListener {
* @return
* @throws IOException
*/
public Note createNote(List<String> interpreterIds) throws IOException {
public Note createNote(List<String> interpreterIds, AuthenticationInfo subject)
throws IOException {
NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory);
Note note = new Note(
notebookRepo,
@ -166,10 +168,11 @@ public class Notebook implements NoteEventListener {
}
if (interpreterIds != null) {
bindInterpretersToNote(note.id(), interpreterIds);
note.putDefaultReplName();
}
notebookIndex.addIndexDoc(note);
note.persist();
note.persist(subject);
fireNoteCreateEvent(note);
return note;
}
@ -198,7 +201,8 @@ public class Notebook implements NoteEventListener {
* @return notebook ID
* @throws IOException
*/
public Note importNote(String sourceJson, String noteName) throws IOException {
public Note importNote(String sourceJson, String noteName, AuthenticationInfo subject)
throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
@ -207,7 +211,7 @@ public class Notebook implements NoteEventListener {
Note newNote;
try {
Note oldNote = gson.fromJson(reader, Note.class);
newNote = createNote();
newNote = createNote(subject);
if (noteName != null)
newNote.setName(noteName);
else
@ -217,7 +221,7 @@ public class Notebook implements NoteEventListener {
newNote.addCloneParagraph(p);
}
newNote.persist();
newNote.persist(subject);
} catch (IOException e) {
logger.error(e.toString(), e);
throw e;
@ -233,14 +237,14 @@ public class Notebook implements NoteEventListener {
* @return noteId
* @throws IOException, CloneNotSupportedException, IllegalArgumentException
*/
public Note cloneNote(String sourceNoteID, String newNoteName) throws
public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject) throws
IOException, CloneNotSupportedException, IllegalArgumentException {
Note sourceNote = getNote(sourceNoteID);
if (sourceNote == null) {
throw new IllegalArgumentException(sourceNoteID + "not found");
}
Note newNote = createNote();
Note newNote = createNote(subject);
if (newNoteName != null) {
newNote.setName(newNoteName);
}
@ -254,7 +258,7 @@ public class Notebook implements NoteEventListener {
}
notebookIndex.addIndexDoc(newNote);
newNote.persist();
newNote.persist(subject);
return newNote;
}
@ -299,7 +303,7 @@ public class Notebook implements NoteEventListener {
}
}
public void removeNote(String id) {
public void removeNote(String id, AuthenticationInfo subject) {
Note note;
synchronized (notes) {
@ -351,21 +355,22 @@ public class Notebook implements NoteEventListener {
fireNoteRemoveEvent(note);
try {
note.unpersist();
note.unpersist(subject);
} catch (IOException e) {
logger.error(e.toString(), e);
}
}
public void checkpointNote(String noteId, String checkpointMessage) throws IOException {
notebookRepo.checkpoint(noteId, checkpointMessage);
public void checkpointNote(String noteId, String checkpointMessage, AuthenticationInfo subject)
throws IOException {
notebookRepo.checkpoint(noteId, checkpointMessage, subject);
}
@SuppressWarnings("rawtypes")
private Note loadNoteFromRepo(String id) {
private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
Note note = null;
try {
note = notebookRepo.get(id);
note = notebookRepo.get(id, subject);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
@ -442,10 +447,10 @@ public class Notebook implements NoteEventListener {
}
private void loadAllNotes() throws IOException {
List<NoteInfo> noteInfos = notebookRepo.list();
List<NoteInfo> noteInfos = notebookRepo.list(null);
for (NoteInfo info : noteInfos) {
loadNoteFromRepo(info.getId());
loadNoteFromRepo(info.getId(), null);
}
}
@ -456,7 +461,7 @@ public class Notebook implements NoteEventListener {
* @return
* @throws IOException
*/
public void reloadAllNotes() throws IOException {
public void reloadAllNotes(AuthenticationInfo subject) throws IOException {
synchronized (notes) {
notes.clear();
}
@ -468,9 +473,9 @@ public class Notebook implements NoteEventListener {
}
}
List<NoteInfo> noteInfos = notebookRepo.list();
List<NoteInfo> noteInfos = notebookRepo.list(subject);
for (NoteInfo info : noteInfos) {
loadNoteFromRepo(info.getId());
loadNoteFromRepo(info.getId(), subject);
}
}
@ -578,12 +583,12 @@ public class Notebook implements NoteEventListener {
}
public List<Map<String, Object>> getJobListforNotebook(boolean needsReload,
long lastUpdateServerUnixTime) {
long lastUpdateServerUnixTime, AuthenticationInfo subject) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
if (needsReload) {
try {
reloadAllNotes();
reloadAllNotes(subject);
} catch (IOException e) {
logger.error("Fail to reload notes from repository");
}
@ -673,7 +678,7 @@ public class Notebook implements NoteEventListener {
Note note = notebook.getNote(noteId);
note.runAll();
while (!note.getLastParagraph().isTerminated()) {
while (!note.isTerminated()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {

25
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
@ -70,7 +71,7 @@ public class AzureNotebookRepo implements NotebookRepo {
}
@Override
public List<NoteInfo> list() throws IOException {
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<NoteInfo>();
NoteInfo info = null;
@ -134,12 +135,12 @@ public class AzureNotebookRepo implements NotebookRepo {
}
@Override
public Note get(String noteId) throws IOException {
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
return getNote(noteId);
}
@Override
public void save(Note note) throws IOException {
public void save(Note note, AuthenticationInfo subject) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
@ -186,7 +187,7 @@ public class AzureNotebookRepo implements NotebookRepo {
}
@Override
public void remove(String noteId) throws IOException {
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
try {
CloudFileDirectory dir = rootDir.getDirectoryReference(noteId);
@ -205,8 +206,22 @@ public class AzureNotebookRepo implements NotebookRepo {
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
throws IOException {
// no-op
LOG.info("Checkpoint feature isn't supported in {}", this.getClass().toString());
return null;
}
@Override
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
// Auto-generated method stub
return null;
}
@Override
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
// Auto-generated method stub
return null;
}
}

27
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
View file

@ -23,6 +23,7 @@ import java.util.List;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoHeadException;
@ -46,7 +47,7 @@ import com.google.common.collect.Lists;
*
* TODO(bzz): add default .gitignore
*/
public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoVersioned {
public class GitNotebookRepo extends VFSNotebookRepo {
private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class);
private String localPath;
@ -65,47 +66,50 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoVers
}
@Override
public synchronized void save(Note note) throws IOException {
super.save(note);
public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
super.save(note, subject);
}
/* implemented as git add+commit
* @param pattern is the noteId
* @param commitMessage is a commit message (checkpoint name)
* @param commitMessage is a commit message (checkpoint message)
* (non-Javadoc)
* @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String)
*/
@Override
public void checkpoint(String pattern, String commitMessage) {
public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) {
Revision revision = null;
try {
List<DiffEntry> gitDiff = git.diff().call();
if (!gitDiff.isEmpty()) {
LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff);
DirCache added = git.add().addFilepattern(pattern).call();
LOG.debug("{} changes are about to be commited", added.getEntryCount());
git.commit().setMessage(commitMessage).call();
RevCommit commit = git.commit().setMessage(commitMessage).call();
revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime());
} else {
LOG.debug("No changes found {}", pattern);
}
} catch (GitAPIException e) {
LOG.error("Failed to add+comit {} to Git", pattern, e);
}
return revision;
}
@Override
public Note get(String noteId, String rev) throws IOException {
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
//TODO(bzz): something like 'git checkout rev', that will not change-the-world though
return super.get(noteId);
return super.get(noteId, subject);
}
@Override
public List<Rev> history(String noteId) {
List<Rev> history = Lists.newArrayList();
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
List<Revision> history = Lists.newArrayList();
LOG.debug("Listing history for {}:", noteId);
try {
Iterable<RevCommit> logs = git.log().addPath(noteId).call();
for (RevCommit log: logs) {
history.add(new Rev(log.getName(), log.getCommitTime()));
history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime()));
LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage());
}
} catch (NoHeadException e) {
@ -131,5 +135,4 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoVers
this.git = git;
}
}

84
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java
View file

@ -23,15 +23,44 @@ import java.util.List;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.user.AuthenticationInfo;
/**
* Notebook repository (persistence layer) abstraction
*/
public interface NotebookRepo {
@ZeppelinApi public List<NoteInfo> list() throws IOException;
@ZeppelinApi public Note get(String noteId) throws IOException;
@ZeppelinApi public void save(Note note) throws IOException;
@ZeppelinApi public void remove(String noteId) throws IOException;
/**
* Lists notebook information about all notebooks in storage.
* @param subject contains user information.
* @return
* @throws IOException
*/
@ZeppelinApi public List<NoteInfo> list(AuthenticationInfo subject) throws IOException;
/**
* Get the notebook with the given id.
* @param noteId is notebook id.
* @param subject contains user information.
* @return
* @throws IOException
*/
@ZeppelinApi public Note get(String noteId, AuthenticationInfo subject) throws IOException;
/**
* Save given note in storage
* @param note is the note itself.
* @param subject contains user information.
* @throws IOException
*/
@ZeppelinApi public void save(Note note, AuthenticationInfo subject) throws IOException;
/**
* Remove note with given id.
* @param noteId is the note id.
* @param subject contains user information.
* @throws IOException
*/
@ZeppelinApi public void remove(String noteId, AuthenticationInfo subject) throws IOException;
/**
* Release any underlying resources
@ -39,7 +68,50 @@ public interface NotebookRepo {
@ZeppelinApi public void close();
/**
* chekpoint (versioning) for notebooks (optional)
* Versioning API (optional, preferred to have).
*/
@ZeppelinApi public void checkpoint(String noteId, String checkPointName) throws IOException;
/**
* chekpoint (set revision) for notebook.
* @param noteId Id of the Notebook
* @param checkpointMsg message description of the checkpoint
* @return Rev
* @throws IOException
*/
@ZeppelinApi public Revision checkpoint(String noteId, String checkpointMsg,
AuthenticationInfo subject) throws IOException;
/**
* Get particular revision of the Notebook.
*
* @param noteId Id of the Notebook
* @param rev revision of the Notebook
* @return a Notebook
* @throws IOException
*/
@ZeppelinApi public Note get(String noteId, Revision rev, AuthenticationInfo subject)
throws IOException;
/**
* List of revisions of the given Notebook.
*
* @param noteId id of the Notebook
* @return list of revisions
*/
@ZeppelinApi public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject);
/**
* Represents the 'Revision' a point in life of the notebook
*/
static class Revision {
public Revision(String revId, String message, int time) {
this.revId = revId;
this.message = message;
this.time = time;
}
public String revId;
public String message;
public int time;
}
}

72
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
View file

@ -31,6 +31,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -114,37 +115,37 @@ public class NotebookRepoSync implements NotebookRepo {
* Lists Notebooks from the first repository
*/
@Override
public List<NoteInfo> list() throws IOException {
return getRepo(0).list();
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
return getRepo(0).list(subject);
}
/* list from specific repo (for tests) */
List<NoteInfo> list(int repoIndex) throws IOException {
return getRepo(repoIndex).list();
List<NoteInfo> list(int repoIndex, AuthenticationInfo subject) throws IOException {
return getRepo(repoIndex).list(subject);
}
/**
* Returns from Notebook from the first repository
*/
@Override
public Note get(String noteId) throws IOException {
return getRepo(0).get(noteId);
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
return getRepo(0).get(noteId, subject);
}
/* get note from specific repo (for tests) */
Note get(int repoIndex, String noteId) throws IOException {
return getRepo(repoIndex).get(noteId);
Note get(int repoIndex, String noteId, AuthenticationInfo subject) throws IOException {
return getRepo(repoIndex).get(noteId, subject);
}
/**
* Saves to all repositories
*/
@Override
public void save(Note note) throws IOException {
getRepo(0).save(note);
public void save(Note note, AuthenticationInfo subject) throws IOException {
getRepo(0).save(note, subject);
if (getRepoCount() > 1) {
try {
getRepo(1).save(note);
getRepo(1).save(note, subject);
}
catch (IOException e) {
LOG.info(e.getMessage() + ": Failed to write to secondary storage");
@ -153,14 +154,14 @@ public class NotebookRepoSync implements NotebookRepo {
}
/* save note to specific repo (for tests) */
void save(int repoIndex, Note note) throws IOException {
getRepo(repoIndex).save(note);
void save(int repoIndex, Note note, AuthenticationInfo subject) throws IOException {
getRepo(repoIndex).save(note, subject);
}
@Override
public void remove(String noteId) throws IOException {
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
for (NotebookRepo repo : repos) {
repo.remove(noteId);
repo.remove(noteId, subject);
}
/* TODO(khalid): handle case when removing from secondary storage fails */
}
@ -174,8 +175,8 @@ public class NotebookRepoSync implements NotebookRepo {
LOG.info("Sync started");
NotebookRepo srcRepo = getRepo(sourceRepoIndex);
NotebookRepo dstRepo = getRepo(destRepoIndex);
List <NoteInfo> srcNotes = srcRepo.list();
List <NoteInfo> dstNotes = dstRepo.list();
List <NoteInfo> srcNotes = srcRepo.list(null);
List <NoteInfo> dstNotes = dstRepo.list(null);
Map<String, List<String>> noteIDs = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo);
List<String> pushNoteIDs = noteIDs.get(pushKey);
@ -210,7 +211,7 @@ public class NotebookRepoSync implements NotebookRepo {
private void pushNotes(List<String> ids, NotebookRepo localRepo,
NotebookRepo remoteRepo) throws IOException {
for (String id : ids) {
remoteRepo.save(localRepo.get(id));
remoteRepo.save(localRepo.get(id, null), null);
}
}
@ -242,8 +243,8 @@ public class NotebookRepoSync implements NotebookRepo {
dnote = containsID(destNotes, snote.getId());
if (dnote != null) {
/* note exists in source and destination storage systems */
sdate = lastModificationDate(sourceRepo.get(snote.getId()));
ddate = lastModificationDate(destRepo.get(dnote.getId()));
sdate = lastModificationDate(sourceRepo.get(snote.getId(), null));
ddate = lastModificationDate(destRepo.get(dnote.getId(), null));
if (sdate.after(ddate)) {
/* source contains more up to date note - push */
pushIDs.add(snote.getId());
@ -354,13 +355,17 @@ public class NotebookRepoSync implements NotebookRepo {
//checkpoint to all available storages
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
throws IOException {
int repoCount = getRepoCount();
int repoBound = Math.min(repoCount, getMaxRepoNum());
int errorCount = 0;
String errorMessage = "";
for (int i = 0; i < Math.min(repoCount, getMaxRepoNum()); i++) {
List<Revision> allRepoCheckpoints = new ArrayList<Revision>();
Revision rev = null;
for (int i = 0; i < repoBound; i++) {
try {
getRepo(i).checkpoint(noteId, checkPointName);
allRepoCheckpoints.add(getRepo(i).checkpoint(noteId, checkpointMsg, subject));
} catch (IOException e) {
LOG.warn("Couldn't checkpoint in {} storage with index {} for note {}",
getRepo(i).getClass().toString(), i, noteId);
@ -370,9 +375,28 @@ public class NotebookRepoSync implements NotebookRepo {
}
}
// throw exception if failed to commit for all initialized repos
if (errorCount == Math.min(repoCount, getMaxRepoNum())) {
if (errorCount == repoBound) {
throw new IOException(errorMessage);
}
if (allRepoCheckpoints.size() > 0) {
rev = allRepoCheckpoints.get(0);
// if failed to checkpoint on first storage, then return result on second
if (allRepoCheckpoints.size() > 1 && rev == null) {
rev = allRepoCheckpoints.get(1);
}
}
return rev;
}
@Override
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
// Auto-generated method stub
return null;
}
@Override
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
// Auto-generated method stub
return null;
}
}

60
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoVersioned.java
View file

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook.repo;
import java.io.IOException;
import java.util.List;
import org.apache.zeppelin.notebook.Note;
/**
* Notebook repository w/ versions
*/
public interface NotebookRepoVersioned extends NotebookRepo {
/**
* Get particular revision of the Notebooks
*
* @param noteId Id of the Notebook
* @param rev revision of the Notebook
* @return a Notebook
* @throws IOException
*/
public Note get(String noteId, String rev) throws IOException;
/**
* List of revisions of the given Notebook
*
* @param noteId id of the Notebook
* @return list of revisions
*/
public List<Rev> history(String noteId);
/**
* Represents the 'Revision' a point in life of the notebook
*/
static class Rev {
public Rev(String name, int time) {
this.name = name;
this.time = time;
}
String name;
int time;
}
}

28
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -38,6 +38,7 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -136,7 +137,7 @@ public class S3NotebookRepo implements NotebookRepo {
}
@Override
public List<NoteInfo> list() throws IOException {
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
@ -196,12 +197,12 @@ public class S3NotebookRepo implements NotebookRepo {
}
@Override
public Note get(String noteId) throws IOException {
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json");
}
@Override
public void save(Note note) throws IOException {
public void save(Note note, AuthenticationInfo subject) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
@ -224,7 +225,7 @@ public class S3NotebookRepo implements NotebookRepo {
}
@Override
public void remove(String noteId) throws IOException {
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
.withBucketName(bucketName).withPrefix(key);
@ -249,8 +250,21 @@ public class S3NotebookRepo implements NotebookRepo {
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
// no-op
LOG.info("Checkpoint feature isn't supported in {}", this.getClass().toString());
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
throws IOException {
// Auto-generated method stub
return null;
}
@Override
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
// Auto-generated method stub
return null;
}
@Override
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
// Auto-generated method stub
return null;
}
}

28
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -41,6 +41,7 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -108,7 +109,7 @@ public class VFSNotebookRepo implements NotebookRepo {
}
@Override
public List<NoteInfo> list() throws IOException {
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
FileObject rootDir = getRootDir();
FileObject[] children = rootDir.getChildren();
@ -192,7 +193,7 @@ public class VFSNotebookRepo implements NotebookRepo {
}
@Override
public Note get(String noteId) throws IOException {
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
FileObject rootDir = fsManager.resolveFile(getPath("/"));
FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
@ -214,7 +215,7 @@ public class VFSNotebookRepo implements NotebookRepo {
}
@Override
public synchronized void save(Note note) throws IOException {
public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
@ -240,7 +241,7 @@ public class VFSNotebookRepo implements NotebookRepo {
}
@Override
public void remove(String noteId) throws IOException {
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
FileObject rootDir = fsManager.resolveFile(getPath("/"));
FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
@ -263,9 +264,22 @@ public class VFSNotebookRepo implements NotebookRepo {
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
// no-op
logger.info("Checkpoint feature isn't supported in {}", this.getClass().toString());
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
throws IOException {
// Auto-generated method stub
return null;
}
@Override
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
// Auto-generated method stub
return null;
}
@Override
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
// Auto-generated method stub
return null;
}
}

27
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.zeppelinhub.rest.ZeppelinhubRestApiHandler;
import org.apache.zeppelin.notebook.repo.zeppelinhub.websocket.Client;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -141,7 +142,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
@Override
public List<NoteInfo> list() throws IOException {
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
String response = restApiClient.asyncGet("");
List<NoteInfo> notes = GSON.fromJson(response, new TypeToken<List<NoteInfo>>() {}.getType());
if (notes == null) {
@ -152,7 +153,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
@Override
public Note get(String noteId) throws IOException {
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
if (StringUtils.isBlank(noteId)) {
return EMPTY_NOTE;
}
@ -167,7 +168,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
@Override
public void save(Note note) throws IOException {
public void save(Note note, AuthenticationInfo subject) throws IOException {
if (note == null) {
throw new IOException("Zeppelinhub failed to save empty note");
}
@ -177,7 +178,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
@Override
public void remove(String noteId) throws IOException {
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
restApiClient.asyncDel(noteId);
LOG.info("ZeppelinHub REST API removing note {} ", noteId);
}
@ -188,8 +189,22 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
throws IOException {
// Auto-generated method stub
return null;
}
@Override
public Note get(String noteId, Revision rev, AuthenticationInfo subject) throws IOException {
// Auto-generated method stub
return null;
}
@Override
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
// Auto-generated method stub
return null;
}
}

58
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
View file

@ -32,28 +32,44 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<String, Object>();
public MockInterpreter1(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterResult result;
if ("getId".equals(st)) {
// get unique id of this interpreter instance
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
} else {
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
public MockInterpreter1(Properties property) {
super(property);
}
boolean open;
@Override
public void open() {
open = true;
}
@Override
public void close() {
open = false;
}
public boolean isOpen() {
return open;
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterResult result;
if ("getId".equals(st)) {
// get unique id of this interpreter instance
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
} else if (st.startsWith("sleep")) {
try {
Thread.sleep(Integer.parseInt(st.split(" ")[1]));
} catch (InterruptedException e) {
// nothing to do
}
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
} else {
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st);
}
if (context.getResourcePool() != null) {
context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result);

16
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
View file

@ -36,14 +36,23 @@ public class MockInterpreter2 extends Interpreter{
super(property);
}
boolean open;
@Override
public void open() {
open = true;
}
@Override
public void close() {
open = false;
}
public boolean isOpen() {
return open;
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterResult result;
@ -51,6 +60,13 @@ public class MockInterpreter2 extends Interpreter{
if ("getId".equals(st)) {
// get unique id of this interpreter instance
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
} else if (st.startsWith("sleep")) {
try {
Thread.sleep(Integer.parseInt(st.split(" ")[1]));
} catch (InterruptedException e) {
// nothing to do
}
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
} else {
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
}

74
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
View file

@ -17,7 +17,11 @@
package org.apache.zeppelin.notebook;
import com.google.common.base.Optional;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
@ -26,6 +30,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.*;
@ -94,4 +99,73 @@ public class NoteTest {
assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText());
assertEquals("Change paragraph text", pText, pCaptor.getValue().getText());
}
@Test
public void putDefaultReplNameIfInterpreterSettingAbsent() {
when(replLoader.getDefaultInterpreterSetting())
.thenReturn(Optional.<InterpreterSetting>absent());
Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
note.putDefaultReplName();
assertEquals(StringUtils.EMPTY, note.getLastReplName());
assertEquals(StringUtils.EMPTY, note.getLastInterpreterName());
}
@Test
public void putDefaultReplNameIfInterpreterSettingPresent() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(replLoader.getDefaultInterpreterSetting())
.thenReturn(Optional.of(interpreterSetting));
Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
note.putDefaultReplName();
assertEquals("spark", note.getLastReplName());
assertEquals("%spark", note.getLastInterpreterName());
}
@Test
public void addParagraphWithLastReplName() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(replLoader.getDefaultInterpreterSetting())
.thenReturn(Optional.of(interpreterSetting));
Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
note.putDefaultReplName(); //set lastReplName
Paragraph p = note.addParagraph();
assertEquals("%spark ", p.getText());
}
@Test
public void insertParagraphWithLastReplName() {
InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class);
when(interpreterSetting.getName()).thenReturn("spark");
when(replLoader.getDefaultInterpreterSetting())
.thenReturn(Optional.of(interpreterSetting));
Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
note.putDefaultReplName(); //set lastReplName
Paragraph p = note.insertParagraph(note.getParagraphs().size());
assertEquals("%spark ", p.getText());
}
@Test
public void setLastReplName() {
String paragraphId = "HelloWorld";
Note note = Mockito.spy(new Note(repo, replLoader, jobListenerFactory, index, credentials));
Paragraph mockParagraph = Mockito.mock(Paragraph.class);
when(note.getParagraph(paragraphId)).thenReturn(mockParagraph);
when(mockParagraph.getRequiredReplName()).thenReturn("spark");
note.setLastReplName(paragraphId);
assertEquals("spark", note.getLastReplName());
}
}

119
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -17,11 +17,7 @@
package org.apache.zeppelin.notebook;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import java.io.File;
@ -110,7 +106,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
// run with defatul repl
@ -149,8 +145,8 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notes.size(), 0);
// load copied notebook on memory when reloadAllNotes() is called
Note copiedNote = notebookRepo.get("2A94M5J1Z");
notebook.reloadAllNotes();
Note copiedNote = notebookRepo.get("2A94M5J1Z", null);
notebook.reloadAllNotes(null);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 1);
assertEquals(notes.get(0).id(), copiedNote.id());
@ -165,14 +161,14 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notes.size(), 1);
// delete notebook from notebook list when reloadAllNotes() is called
notebook.reloadAllNotes();
notebook.reloadAllNotes(null);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 0);
}
@Test
public void testPersist() throws IOException, SchedulerException, RepositoryException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
// run with default repl
Paragraph p1 = note.addParagraph();
@ -180,7 +176,7 @@ public class NotebookTest implements JobListenerFactory{
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
note.persist();
note.persist(null);
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
@ -190,7 +186,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testClearParagraphOutput() throws IOException, SchedulerException{
Note note = notebook.createNote();
Note note = notebook.createNote(null);
Paragraph p1 = note.addParagraph();
Map config = p1.getConfig();
config.put("enabled", true);
@ -208,7 +204,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
// p1
@ -241,13 +237,13 @@ public class NotebookTest implements JobListenerFactory{
assertNull(p2.getResult());
assertEquals("repl1: p3", p3.getResult().message());
notebook.removeNote(note.getId());
notebook.removeNote(note.getId(), null);
}
@Test
public void testSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph();
@ -279,48 +275,59 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph();
Map config = new HashMap<String, Object>();
p.setConfig(config);
p.setText("p1");
p.setText("sleep 1000");
Paragraph p2 = note.addParagraph();
p2.setConfig(config);
p2.setText("%mock2 sleep 500");
// set cron scheduler, once a second
config = note.getConfig();
config.put("enabled", true);
config.put("cron", "* * * * * ?");
config.put("releaseresource", "true");
config.put("cron", "1/3 * * * * ?");
config.put("releaseresource", true);
note.setConfig(config);
notebook.refreshCron(note.id());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
Date dateFinished = p.getDateFinished();
assertNotNull(dateFinished);
// restart interpreter
for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) {
notebook.getInterpreterFactory().restart(setting.id());
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
((LazyOpenInterpreter) note.getNoteReplLoader().get("mock1")).getInnerInterpreter())
.getInnerInterpreter()));
MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
((LazyOpenInterpreter) note.getNoteReplLoader().get("mock2")).getInnerInterpreter())
.getInnerInterpreter()));
// wait until interpreters are started
while (!mock1.isOpen() || !mock2.isOpen()) {
Thread.yield();
}
Thread.sleep(1000);
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
// wait until interpreters are closed
while (mock1.isOpen() || mock2.isOpen()) {
Thread.yield();
}
assertNotEquals(dateFinished, p.getDateFinished());
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.id());
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
assertNotNull(p2.getDateFinished());
}
@Test
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph();
@ -329,7 +336,7 @@ public class NotebookTest implements JobListenerFactory{
while(p.isTerminated()==false || p.getResult()==null) Thread.yield();
p.setStatus(Status.RUNNING);
Note cloneNote = notebook.cloneNote(note.getId(), "clone note");
Note cloneNote = notebook.cloneNote(note.getId(), "clone note", null);
Paragraph cp = cloneNote.paragraphs.get(0);
assertEquals(cp.getStatus(), Status.READY);
@ -341,7 +348,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
@ -362,7 +369,7 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(1, ResourcePoolUtils.getAllResources().size());
// remove note
notebook.removeNote(note.id());
notebook.removeNote(note.id(), null);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
}
@ -370,7 +377,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = note.getNoteReplLoader()
@ -389,7 +396,7 @@ public class NotebookTest implements JobListenerFactory{
registry.add("o3", "object3", null, null);
// remove notebook
notebook.removeNote(note.id());
notebook.removeNote(note.id(), null);
// notebook scope or paragraph scope object should be removed
assertNull(registry.get("o1", note.id(), null));
@ -403,7 +410,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = note.getNoteReplLoader()
@ -436,7 +443,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = note.getNoteReplLoader()
@ -457,13 +464,13 @@ public class NotebookTest implements JobListenerFactory{
// local and global scope object should be removed
assertNull(registry.get("o1", note.id(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.id());
notebook.removeNote(note.id(), null);
}
@Test
public void testPermissions() throws IOException {
// create a note and a paragraph
Note note = notebook.createNote();
Note note = notebook.createNote(null);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
// empty owners, readers and writers means note is public
assertEquals(notebookAuthorization.isOwner(note.id(),
@ -502,13 +509,13 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notebookAuthorization.isReader(note.id(),
new HashSet<String>(Arrays.asList("user3"))), true);
notebook.removeNote(note.id());
notebook.removeNote(note.id(), null);
}
@Test
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
ArrayList<Paragraph> paragraphs = new ArrayList<>();
@ -545,7 +552,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException {
// create a notes
Note note1 = notebook.createNote();
Note note1 = notebook.createNote(null);
Paragraph p1 = note1.addParagraph();
p1.setText("getId");
@ -560,8 +567,8 @@ public class NotebookTest implements JobListenerFactory{
InterpreterResult result = p1.getResult();
// remove note and recreate
notebook.removeNote(note1.getId());
note1 = notebook.createNote();
notebook.removeNote(note1.getId(), null);
note1 = notebook.createNote(null);
p1 = note1.addParagraph();
p1.setText("getId");
@ -569,16 +576,16 @@ public class NotebookTest implements JobListenerFactory{
while (p1.getStatus() != Status.FINISHED) Thread.yield();
assertNotEquals(p1.getResult().message(), result.message());
notebook.removeNote(note1.getId());
notebook.removeNote(note1.getId(), null);
}
@Test
public void testPerSessionInterpreter() throws IOException {
// create two notes
Note note1 = notebook.createNote();
Note note1 = notebook.createNote(null);
Paragraph p1 = note1.addParagraph();
Note note2 = notebook.createNote();
Note note2 = notebook.createNote(null);
Paragraph p2 = note2.addParagraph();
p1.setText("getId");
@ -609,14 +616,14 @@ public class NotebookTest implements JobListenerFactory{
assertNotEquals(p1.getResult().message(), p2.getResult().message());
notebook.removeNote(note1.getId());
notebook.removeNote(note2.getId());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note2.getId(), null);
}
@Test
public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IOException {
// create a notes
Note note1 = notebook.createNote();
Note note1 = notebook.createNote(null);
Paragraph p1 = note1.addParagraph();
p1.setText("getId");
@ -641,7 +648,7 @@ public class NotebookTest implements JobListenerFactory{
assertNotEquals(result.message(), p1.getResult().message());
notebook.removeNote(note1.getId());
notebook.removeNote(note1.getId(), null);
}
@Test
@ -707,7 +714,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testNormalizeNoteName() throws IOException {
// create a notes
Note note1 = notebook.createNote();
Note note1 = notebook.createNote(null);
note1.setName("MyNote");
assertEquals(note1.getName(), "MyNote");
@ -727,7 +734,7 @@ public class NotebookTest implements JobListenerFactory{
note1.setName("\\\\\\MyNote///sub");
assertEquals(note1.getName(), "/MyNote/sub");
notebook.removeNote(note1.getId());
notebook.removeNote(note1.getId(), null);
}
private void delete(File file){

28
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
View file

@ -32,7 +32,7 @@ import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.repo.NotebookRepoVersioned.Rev;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.DiffEntry;
@ -97,7 +97,7 @@ public class GitNotebookRepoTest {
assertThat(git).isNotNull();
assertThat(dotGit.exists()).isEqualTo(true);
assertThat(notebookRepo.list()).isNotEmpty();
assertThat(notebookRepo.list(null)).isNotEmpty();
List<DiffEntry> diff = git.diff().call();
// no commit, diff isn't empty
@ -108,10 +108,10 @@ public class GitNotebookRepoTest {
public void showNotebookHistory() throws GitAPIException, IOException {
//given
notebookRepo = new GitNotebookRepo(conf);
assertThat(notebookRepo.list()).isNotEmpty();
assertThat(notebookRepo.list(null)).isNotEmpty();
//when
List<Rev> testNotebookHistory = notebookRepo.history(TEST_NOTE_ID);
List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
//then
//no initial commit, empty history
@ -122,17 +122,17 @@ public class GitNotebookRepoTest {
public void addCheckpoint() throws IOException {
// initial checks
notebookRepo = new GitNotebookRepo(conf);
assertThat(notebookRepo.list()).isNotEmpty();
assertThat(containsNote(notebookRepo.list(), TEST_NOTE_ID)).isTrue();
assertThat(notebookRepo.history(TEST_NOTE_ID)).isEmpty();
assertThat(notebookRepo.list(null)).isNotEmpty();
assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
notebookRepo.checkpoint(TEST_NOTE_ID, "first commit");
List<Rev> notebookHistoryBefore = notebookRepo.history(TEST_NOTE_ID);
assertThat(notebookRepo.history(TEST_NOTE_ID)).isNotEmpty();
notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null);
List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isNotEmpty();
int initialCount = notebookHistoryBefore.size();
// add changes to note
Note note = notebookRepo.get(TEST_NOTE_ID);
Note note = notebookRepo.get(TEST_NOTE_ID, null);
Paragraph p = note.addParagraph();
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
@ -140,11 +140,11 @@ public class GitNotebookRepoTest {
p.setText("%md checkpoint test text");
// save and checkpoint note
notebookRepo.save(note);
notebookRepo.checkpoint(TEST_NOTE_ID, "second commit");
notebookRepo.save(note, null);
notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null);
// see if commit is added
List<Rev> notebookHistoryAfter = notebookRepo.history(TEST_NOTE_ID);
List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1);
}

80
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -118,16 +118,16 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0).size());
assertEquals(0, notebookRepoSync.list(1).size());
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
/* create note */
Note note = notebookSync.createNote();
Note note = notebookSync.createNote(null);
// check that automatically saved on both storages
assertEquals(1, notebookRepoSync.list(0).size());
assertEquals(1, notebookRepoSync.list(1).size());
assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
assertEquals(notebookRepoSync.list(0, null).get(0).getId(),notebookRepoSync.list(1, null).get(0).getId());
}
@ -135,22 +135,22 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void testSyncOnDelete() throws IOException {
/* create note */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0).size());
assertEquals(0, notebookRepoSync.list(1).size());
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
Note note = notebookSync.createNote();
Note note = notebookSync.createNote(null);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0).size());
assertEquals(1, notebookRepoSync.list(1).size());
assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
assertEquals(notebookRepoSync.list(0, null).get(0).getId(),notebookRepoSync.list(1, null).get(0).getId());
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId());
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), null);
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0).size());
assertEquals(0, notebookRepoSync.list(1).size());
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
}
@ -158,7 +158,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void testSyncUpdateMain() throws IOException {
/* create note */
Note note = notebookSync.createNote();
Note note = notebookSync.createNote(null);
Paragraph p1 = note.addParagraph();
Map config = p1.getConfig();
config.put("enabled", true);
@ -170,37 +170,37 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note);
notebookRepoSync.save(0, note, null);
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
/* check paragraph isn't saved to second storage */
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* apply sync */
notebookRepoSync.sync();
/* check whether added to second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* check whether same paragraph id */
assertEquals(p1.getId(), notebookRepoSync.get(0,
notebookRepoSync.list(0).get(0).getId()).getLastParagraph().getId());
notebookRepoSync.list(0, null).get(0).getId(), null).getLastParagraph().getId());
assertEquals(p1.getId(), notebookRepoSync.get(1,
notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId());
notebookRepoSync.list(1, null).get(0).getId(), null).getLastParagraph().getId());
}
@Test
public void testSyncOnReloadedList() throws IOException {
/* check that both storage repos are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0).size());
assertEquals(0, notebookRepoSync.list(1).size());
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
File srcDir = new File("src/test/resources/2A94M5J1Z");
File destDir = new File(secNotebookDir + "/2A94M5J1Z");
@ -211,13 +211,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
} catch (IOException e) {
LOG.error(e.toString(), e);
}
assertEquals(0, notebookRepoSync.list(0).size());
assertEquals(1, notebookRepoSync.list(1).size());
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
// After reloading notebooks repos should be synchronized
notebookSync.reloadAllNotes();
assertEquals(1, notebookRepoSync.list(0).size());
assertEquals(1, notebookRepoSync.list(1).size());
notebookSync.reloadAllNotes(null);
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
}
@Test
@ -236,15 +236,15 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
// no notes
assertThat(vRepoSync.list().size()).isEqualTo(0);
assertThat(vRepoSync.list(null).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote();
assertThat(vRepoSync.list().size()).isEqualTo(1);
Note note = vNotebookSync.createNote(null);
assertThat(vRepoSync.list(null).size()).isEqualTo(1);
String noteId = vRepoSync.list().get(0).getId();
String noteId = vRepoSync.list(null).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message");
int vCount = gitRepo.history(noteId).size();
vRepoSync.checkpoint(noteId, "checkpoint message", null);
int vCount = gitRepo.revisionHistory(noteId, null).size();
assertThat(vCount).isEqualTo(1);
Paragraph p = note.addParagraph();
@ -254,9 +254,9 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
p.setText("%md checkpoint test");
// save and checkpoint again
vRepoSync.save(note);
vRepoSync.checkpoint(noteId, "checkpoint message 2");
assertThat(gitRepo.history(noteId).size()).isEqualTo(vCount + 1);
vRepoSync.save(note, null);
vRepoSync.checkpoint(noteId, "checkpoint message 2", null);
assertThat(gitRepo.revisionHistory(noteId, null).size()).isEqualTo(vCount + 1);
}
static void delete(File file){

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -93,7 +93,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
@Test
public void testInvalidJsonFile() throws IOException {
// given
int numNotes = notebookRepo.list().size();
int numNotes = notebookRepo.list(null).size();
// when create invalid json file
File testNoteDir = new File(mainNotebookDir, "test");
@ -101,12 +101,12 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
// then
assertEquals(numNotes, notebookRepo.list().size());
assertEquals(numNotes, notebookRepo.list(null).size());
}
@Test
public void testSaveNotebook() throws IOException, InterruptedException {
Note note = notebook.createNote();
Note note = notebook.createNote(null);
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
Paragraph p1 = note.addParagraph();
@ -133,7 +133,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
}
note.setName("SaveTest");
notebookRepo.save(note);
notebookRepo.save(note, null);
assertEquals(note.getName(), "SaveTest");
}
@ -146,7 +146,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
@Override
public void run() {
try {
notebookRepo.save(note);
notebookRepo.save(note, null);
} catch (IOException e) {
LOG.error(e.toString(), e);
}

8
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepoTest.java
View file

@ -123,14 +123,14 @@ public class ZeppelinHubRepoTest {
@Test
public void testGetAllNotes() throws IOException {
List<NoteInfo> notebooks = repo.list();
List<NoteInfo> notebooks = repo.list(null);
assertThat(notebooks).isNotEmpty();
assertThat(notebooks.size()).isEqualTo(3);
}
@Test
public void testGetNote() throws IOException {
Note notebook = repo.get("AAAAA");
Note notebook = repo.get("AAAAA", null);
assertThat(notebook).isNotNull();
assertThat(notebook.id()).isEqualTo("2A94M5J1Z");
}
@ -138,13 +138,13 @@ public class ZeppelinHubRepoTest {
@Test
public void testRemoveNote() throws IOException {
// not suppose to throw
repo.remove("AAAAA");
repo.remove("AAAAA", null);
}
@Test
public void testRemoveNoteError() throws IOException {
// not suppose to throw
repo.remove("BBBBB");
repo.remove("BBBBB", null);
}
}

4
zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
View file

@ -204,7 +204,7 @@ public class LuceneSearchTest {
//when
Paragraph p1 = note1.getLastParagraph();
p1.setText("no no no");
note1.persist();
note1.persist(null);
//then
assertThat(resultForQuery("Notebook1").size()).isEqualTo(1);
@ -228,7 +228,7 @@ public class LuceneSearchTest {
//when
note1.setName("NotebookN");
note1.persist();
note1.persist(null);
//then
assertThat(resultForQuery("Notebook1")).isEmpty();