Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/master' into UserInInterpreterContext

Browse source
This commit is contained in:
Prabhjyot Singh 2016-02-16 21:53:53 +05:30
commit ba91da4121
51 changed files with 1384 additions and 629 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.travis.yml
View file

@ -19,27 +19,31 @@ matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.2.1
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.0
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- "export DISPLAY=:99.0"
@ -53,7 +57,7 @@ before_script:
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
after_failure:
- cat target/rat.txt

10
CONTRIBUTING.md
View file

@ -7,6 +7,10 @@ Contributing to Zeppelin (Source code, Documents, Image, Website) means you agre
2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN)
3. Contribute your patch via Pull Request.
Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it.
For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful.
## Creating a Pull Request
In order to make the review process easier, please follow this template when making a Pull Request:
@ -144,13 +148,13 @@ First of all, you need the Zeppelin source code. The official location for Zeppe
Get the source code on your development machine using git.
```
git clone http://git.apache.org/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 http://git.apache.org/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
or with write access

9
SECURITY-README.md
View file

@ -33,11 +33,8 @@ The scope of this PR is to require credentials to access Zeppelin. To achieve th
Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info).
## Websocket security
Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach:
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
1. The browser on startup requests a ticket through HTTP
2. The Apache Shiro Servlet filter handles the user auth
3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser
All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above).

2
docs/_config.yml
View file

@ -21,6 +21,8 @@ author :
twitter : ASF
feedburner : feedname
ZEPPELIN_VERSION : 0.6.0-incubating-SNAPSHOT
# The production_url is only used when full-domain names are needed
# such as sitemap.txt
# Most places will/should use BASE_PATH to make the urls

7
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -31,6 +31,7 @@
<li role="separator" class="divider"></li>
<!-- li><span><b>Guide</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/dynamicform.html">Dynamic Form</a></li>
<li><a href="{{BASE_PATH}}/manual/publish.html">Publish your Paragraph</a></li>
</ul>
</li>
<li>
@ -45,6 +46,7 @@
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/jdbc.html">JDBC</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/markdown.html">Markdown</a></li>
<li><a href="{{BASE_PATH}}/interpreter/postgresql.html">Postgresql, hawq</a></li>
@ -86,9 +88,12 @@
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontribute.html">How to contribute (code)</a></li>
<li><a href="{{BASE_PATH}}/development/howtocontributewebsite.html">How to contribute (website)</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Shiro Security</b><span></li -->
<li><a href="{{BASE_PATH}}/manual/shiroauthentication.html">Shiro Authentication</a></li>
</ul>
</li>
</ul>
</nav><!--/.navbar-collapse -->
</div>
</div>
</div>

BIN
docs/assets/themes/zeppelin/img/docs-img/copy-the-link.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 185 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 7 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/link-the-paragraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 311 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/your-website.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 425 KiB

BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin-login.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

58
docs/development/howtocontribute.md
View file

@ -5,52 +5,49 @@ description: "How to contribute"
group: development
---
# Contributing to Apache Zeppelin ( Code )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
## Setting up
Here are some tools you will need to build and test Zeppelin.
#### Software Configuration Management ( SCM )
### Setting up
Here are some things you will need to do to build and test Zeppelin.
Since Zeppelin uses Git for it's SCM system, you need git client installed in your development machine.
#### Software Configuration Management(SCM)
Zeppelin uses Git for it's SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` You'll need git client installed in your development machine.
#### Integrated Development Environment(IDE)
#### Integrated Development Environment ( IDE )
You are free to use whatever IDE you prefer, or your favorite command line editor.
#### Build Tools
### Build Tools
To build the code, install
Oracle Java 7
Apache Maven
* Oracle Java 7
* Apache Maven
### Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
## Getting the source code
First of all, you need Zeppelin source code. The official location of Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
#### git access
### git access
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git zeppelin
git clone git://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
You may also want to develop against a specific branch. For example, for branch-0.5.6
```
git clone -b branch-0.1 https://github.com/apache/incubator-zeppelin.git zeppelin
git clone -b branch-0.5.6 git://git.apache.org/incubator-zeppelin.git zeppelin
```
#### Fork repository
If you want not only build Zeppelin but also make changes, then you need to fork Zeppelin repository and make pull request.
If you want not only build Zeppelin but also make any changes, then you need fork [Zeppelin github mirror repository](https://github.com/apache/incubator-zeppelin) and make a pull request.
###Build
@ -67,7 +64,7 @@ mvn install -DskipTests
To build with specific spark / hadoop version
```
mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x
```
### Run Zeppelin server in development mode
@ -76,7 +73,8 @@ mvn install -Dspark.version=1.0.1 -Dhadoop.version=2.2.0
cd zeppelin-server
HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args=""
```
NOTE: make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory otherwise your server build will fail to find the required dependencies in the local repro
> **Note:** Make sure you first run ```mvn clean install -DskipTests``` on your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro.
or use daemon script
@ -84,15 +82,13 @@ or use daemon script
bin/zeppelin-daemon start
```
Server will be run on http://localhost:8080
Server will be run on [http://localhost:8080](http://localhost:8080).
### Generating Thrift Code
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this, but if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch.
To regenerate the code, install thrift-0.9.0 and change directory into Zeppelin source directory. and then run following command
To regenerate the code, install **thrift-0.9.0** and change directory into Zeppelin source directory. and then run following command
```
@ -100,10 +96,10 @@ thrift -out zeppelin-interpreter/src/main/java/ --gen java zeppelin-interpreter/
```
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

37
docs/development/howtocontributewebsite.md
View file

@ -5,62 +5,51 @@ description: "How to contribute (website)"
group: development
---
## IMPORTANT
# Contributing to Apache Zeppelin ( Website )
## IMPORTANT
Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software.
Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License.
## Modifying the website
### Modifying the website
<br />
#### Getting the source code
Website is hosted in 'master' branch under `/docs/` dir.
First of all, you need the website source code. The official location of mirror for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin).
First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
Get the source code on your development machine using git.
```
git clone https://github.com/apache/incubator-zeppelin.git
git clone git://git.apache.org/incubator-zeppelin.git
cd docs
```
<br />
#### Build
To build, you'll need to install some prerequisites.
To build, you'll need to install some prerequisites. Please check 'Build documentation' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build-documentation).
Please check 'Build' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#build)
<br />
#### Run website in development mode
While you're modifying website, you'll want to see preview of it.
While you're modifying website, you'll want to see preview of it. Please check 'Run website' section in [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run-website).
Please check 'Run' section on [docs/README.md](https://github.com/apache/incubator-zeppelin/blob/master/docs/README.md#run)
You'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser.
You'll be able to access it on localhost:4000 with your webbrowser.
#### Making a Pull Request
<br />
#### Pull request
When you're ready, just make a pull-request.
When you are ready, just make a pull-request.
<br />
### Alternative way
## Alternative way
You can directly edit .md files in `/docs/` dir at github's web interface and make pull-request immediatly.
<br />
### JIRA
## JIRA
Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN)
### Stay involved
## Stay involved
Contributors should join the Zeppelin mailing lists.
* [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/)

8
docs/index.md
View file

@ -37,7 +37,6 @@ limitations under the License.
<div class="col-md-7"><img class="img-responsive" style="border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/notebook.png" /></div>
</div>
<br />
### Multiple language backend
@ -86,7 +85,6 @@ With simple drag and drop Zeppelin aggeregates the values and display them in pi
</div>
Learn more about Zeppelin's Display system. ( [text](./displaysystem/display.html), [html](./displaysystem/display.html#html), [table](./displaysystem/table.html), [angular](./displaysystem/angular.html) )
<br />
### Dynamic forms
@ -96,7 +94,6 @@ Zeppelin can dynamically create some input forms into your notebook.
Learn more about [Dynamic Forms](./manual/dynamicform.html).
<br />
### Collaboration
@ -114,6 +111,8 @@ This way, you can easily embed it as an iframe inside of your website.</p>
<img class="img-responsive center-block" src="/assets/themes/zeppelin/img/screenshots/publish.png" />
</div>
If you want to learn more about this feature, please visit [this page](./manual/publish.html).
<br />
### 100% Opensource
@ -124,5 +123,4 @@ Join the [Mailing list](./community.html) and report issues on our [Issue tracke
<br />
### Undergoing Incubation
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.

6
docs/install/install.md
View file

@ -75,6 +75,12 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>zeppelin.server.allowed.origins</td>
<td>*</td>
<td>Enables a way to specify a ',' separated list of allowed origins for rest and websockets. <br /> i.e. http://localhost:8080 </td>
</tr>
<tr>
<td>N/A</td>
<td>zeppelin.anonymous.allowed</td>
<td>true</td>
<td>Anonymous user is allowed by default.</td>
</tr>
<tr>
<td>ZEPPELIN_SERVER_CONTEXT_PATH</td>

46
docs/install/virtual_machine.md
View file

@ -22,11 +22,11 @@ limitations under the License.
## Vagrant Virtual Machine for Apache Zeppelin
The Apache Zeppelin distribution includes a scripts directory
Apache Zeppelin distribution includes a scripts directory
`scripts/vagrant/zeppelin-dev`
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras)
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source. For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras).
####Installing the required components to launch a virtual machine.
@ -34,19 +34,22 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
If you are running Windows and don't yet have python installed, [install Python 2.7.x](https://www.python.org/downloads/release/python-2710/) first.
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
```
sudo easy_install pip
sudo pip install ansible
ansible --version
```
After then, please check whether it reports **ansible version 1.9.2 or higher**.
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
Thats it!
You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
Thats it ! You can now run `vagrant ssh` and this will place you into the guest machines terminal prompt.
If you don't wish to build Zeppelin from scratch, run the z-manager installer script while running in the guest VM:
@ -55,18 +58,17 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
```
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning Zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Cloning the project again may seem counter intuitive, since this script likley originated from the project repository. Consider copying just the vagrant/zeppelin-dev script from the Zeppelin project as a stand alone directory, then once again clone the specific branch you wish to build.
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -74,7 +76,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -117,15 +119,15 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
### [Python Extras](id:pythonextras)
With zeppelin running, Numpy, SciPy, Pandas and Matplotlib will be available. Create a pyspark notebook, and try
With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available. Create a pyspark notebook, and try the below code.
```
```python
%pyspark
import numpy
@ -139,9 +141,9 @@ print "pandas " + pandas.__version__
print "matplotlib " + matplotlib.__version__
```
To Test plotting using matplotlib into a rendered %html SVG image, try
To Test plotting using Matplotlib into a rendered `%html` SVG image, try
```
```python
%pyspark
import matplotlib

6
docs/install/yarn_install.md
View file

@ -138,13 +138,13 @@ Zeppelin can work with multiple versions Spark. A complete list [is available he
## Build
Checkout source code from [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
Checkout source code from [git://git.apache.org/incubator-zeppelin.git](git://git.apache.org/incubator-zeppelin.git).
```bash
cd /home/zeppelin/
git clone https://github.com/apache/incubator-zeppelin.git
git clone git://git.apache.org/incubator-zeppelin.git
```
Zeppelin package is available at /home/zeppelin/incubator-zeppelin after the checkout completes.
Zeppelin package is available at `/home/zeppelin/incubator-zeppelin` after the checkout completes.
### Cluster mode

914
docs/interpreter/cassandra.md
View file

File diff suppressed because it is too large Load diff

224
docs/interpreter/jdbc.md Normal file
View file

@ -0,0 +1,224 @@
---
layout: page
title: "Generic JDBC Interpreter"
description: "JDBC user guide"
group: manual
---
{% include JB/setup %}
## Generic JDBC Interpreter for Apache Zeppelin
This interpreter lets you create a JDBC connection to any data source, by now it has been tested with:
* Postgres
* MySql
* MariaDB
* Redshift
* Hive
If someone else used another database please report how it works to improve functionality.
### Create Interpreter
When create a interpreter by default use PostgreSQL with the next properties:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:postgresql://localhost:5432/</td>
</tr>
<tr>
<td>default.user</td>
<td>gpadmin</td>
</tr>
</table>
It is not necessary to add driver jar to the classpath for PostgreSQL as it is included in Zeppelin.
#### Simple connection
Before creating the interpreter it is necessary to add to the Zeppelin classpath the path of the JDBC you want to use, to do it you must edit the file `zeppelin-daemon.sh` as shown:
```
# Add jdbc connector jar
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/mysql-connector-java-5.1.6.jar"
```
For create the interpreter you need to specify connection parameters as shown in the table.
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>driver name</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc url</td>
</tr>
<tr>
<td>default.user</td>
<td>user name</td>
</tr>
</table>
#### Multiple connections
This JDBC interpreter also allows connections to multiple data sources. For every connection is necessary a prefix for reference in the paragraph this way `%jdbc(prefix)`. Before creating the interpreter it is necessary to add to the Zeppelin classpath all paths to access to each driver's jar file you want to use, to do it you must edit the file `zeppelin-daemon.sh` as following:
```
# Add jdbc connector jar
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/RedshiftJDBC41-1.1.10.1010.jar"
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/mysql-connector-java-5.1.6.jar"
```
You can add all the jars you need to make multiple connections into the same interpreter. To create the interpreter you must specify the parameters, for example we will create two connections to PostgreSQL and Redshift, the respective prefixes are `default` and `redshift`:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>common.max_count</td>
<td>1000</td>
</tr>
<tr>
<td>default.driver</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>default.password</td>
<td>********</td>
</tr>
<tr>
<td>default.url</td>
<td>jdbc:postgresql://localhost:5432/</td>
</tr>
<tr>
<td>default.user</td>
<td>gpadmin</td>
</tr>
<tr>
<td>redshift.driver</td>
<td>com.amazon.redshift.jdbc4.Driver</td>
</tr>
<tr>
<td>redshift.password</td>
<td>********</td>
</tr>
<tr>
<td>redshift.url</td>
<td>jdbc:redshift://examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439</td>
</tr>
<tr>
<td>redshift.user</td>
<td>redshift-user</td>
</tr>
</table>
### Bind to Notebook
In the `Notebook` click on the `settings` icon at the top-right corner. Use select/deselect to specify the interpreters to be used in the `Notebook`.
### More Properties
You can modify the interpreter configuration in the `Interpreter` section. The most common properties are as follows, but you can specify other properties that need to be connected.
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
</tr>
<tr>
<td>{prefix}.url</td>
<td>JDBC URL to connect, the URL must include the name of the database </td>
</tr>
<tr>
<td>{prefix}.user</td>
<td>JDBC user name</td>
</tr>
<tr>
<td>{prefix}.password</td>
<td>JDBC password</td>
</tr>
<tr>
<td>{prefix}.driver</td>
<td>JDBC driver name.</td>
</tr>
<tr>
<td>common.max_result</td>
<td>Max number of SQL result to display to prevent the browser overload. This is common properties for all connections</td>
</tr>
</table>
To develop this functionality use this [method](http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html#getConnection%28java.lang.String,%20java.util.Properties%29). For example if a connection needs a schema parameter, it would have to add the property as follows:
<table class="table-configuration">
<tr>
<th>name</th>
<th>value</th>
</tr>
<tr>
<td>{prefix}.schema</td>
<td>schema_name</td>
</tr>
</table>
### How to use
#### Reference in paragraph
Start the paragraphs with the `%jdbc`, this will use the `default` prefix for connection. If you want to use other connection you should specify the prefix of it as follows `%jdbc(prefix)`:
```sql
%jdbc
SELECT * FROM db_name;
```
or
```sql
%jdbc(prefix)
SELECT * FROM db_name;
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
%jdbc(prefix)
SELECT name, country, performer
FROM demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
```
### Bugs & Contacts
If you find a bug for this interpreter, please create a [JIRA]( https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN) ticket.

24
docs/interpreter/spark.md
View file

@ -262,3 +262,27 @@ select * from ${table=defaultTableName} where text like '%${search}%'
```
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
## Setting up Zeppelin with Kerberos
Logical setup with Zeppelin, Kerberos Distribution Center (KDC), and Spark on YARN:
<img src="../assets/themes/zeppelin/img/docs-img/kdc_zeppelin.png">
####Configuration Setup
1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf.
This is to make the server communicate with KDC.
2. Set SPARK\_HOME in [ZEPPELIN\_HOME]/conf/zeppelin-env.sh to use spark-submit
( Additionally, you might have to set “export HADOOP\_CONF\_DIR=/etc/hadoop/conf” )
3. Add the two properties below to spark configuration ( [SPARK_HOME]/conf/spark-defaults.conf ):
spark.yarn.principal
spark.yarn.keytab
> **NOTE:** If you do not have access to the above spark-defaults.conf file, optionally, you may add the lines to the Spark Interpreter through the Interpreter tab in the Zeppelin UI.
4. That's it. Play with Zeppelin !

48
docs/manual/publish.md Normal file
View file

@ -0,0 +1,48 @@
---
layout: page
title: "Publish your Paragraph"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## How can you publish your paragraph ?
Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website.
It's very straightforward. Just use `<iframe>` tag in your page.
> **Warning**: Please use this feature with caution and in a trusted environment only, as Zeppelin entire Webapp could be accessible for whoever visits your website.
### Copy a Paragraph Link
A first step to publish your paragraph result is **Copy a Paragraph Link**.
* After running a paragraph in your Zeppelin notebook, click a gear button located on the right side. Then, click **Link this Paragraph** menu like below image.
<center><img src="../assets/themes/zeppelin/img/docs-img/link-the-paragraph.png" height="100%" width="100%"></center>
* Just copy the provided link.
<center><img src="../assets/themes/zeppelin/img/docs-img/copy-the-link.png" height="100%" width="100%"></center>
### Embed the Paragraph to Your Website
For publishing the copied paragraph, you may use `<iframe>` tag in your website page.
For example,
```
<iframe src="http://< ip-address >:< port >/#/notebook/2B3QSZTKR/paragraph/...?asIframe" height="" width="" ></iframe>
```
Finally, you can show off your beautiful visualization results in your website.
<center><img src="../assets/themes/zeppelin/img/docs-img/your-website.png" height="90%" width="90%"></center>
> **Note**: To embed the paragraph in a website, Zeppelin needs to be reachable by that website.

72
docs/manual/shiroauthentication.md Normal file
View file

@ -0,0 +1,72 @@
---
layout: page
title: "Shiro Security for Apache Zeppelin"
description: ""
group: manual
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Shiro authentication for Apache Zeppelin
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
When you connect to Apache Zeppelin, you will be asked to enter your credentials. Once you logged in, then you have access to all notes including other user's notes.
## Security Setup
You can setup **Zeppelin notebook authentication** in some simple steps.
####1. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
The default status of them is
```
/** = anon
#/** = authcBasic
```
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
```
#/** = anon
/** = authcBasic
```
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
####2. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
####3. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
```
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
####4. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
```
admin = password1
user1 = password2
user2 = password3
```
Those combinations are defined in the `conf/shiro.ini` file.
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/incubator-zeppelin/blob/master/SECURITY-README.md).

11
lens/pom.xml
View file

@ -226,15 +226,4 @@
</plugins>
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
</profiles>
</project>

10
pom.xml
View file

@ -635,6 +635,16 @@
</build>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<!-- Geode can be enabled by -Pgeode. see https://issues.apache.org/jira/browse/ZEPPELIN-375 -->
<profile>
<id>geode</id>

14
scripts/vagrant/zeppelin-dev/README.md
View file

@ -24,9 +24,9 @@ This script requires three applications, [Ansible](http://docs.ansible.com/ansib
*If you are running Windows and don't yet have python installed, install Python 2.7.x* [Python Windows Installer](https://www.python.org/downloads/release/python-2710/)
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
`sudo easy_install pip` then
`sudo pip install ansible`
`ansible --version` should now report version 1.9.2 or higher
3. Install Virtual Box: [Virtual Box Downloads](https://www.virtualbox.org/ "Virtual Box")
4. Type `vagrant up` from within the `/scripts/vagrant/zeppelin-dev` directory
@ -45,7 +45,7 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in
### Building Zeppelin
You can now `git clone https://github.com/apache/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
You can now `git clone git://git.apache.org/incubator-zeppelin.git` into a directory on your host machine, or directly in your virtual machine.
Cloning zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine.
@ -53,7 +53,7 @@ Cloning the project again may seem counter intuitive, since this script likley o
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
`cd /vagrant/incubator-zeppelin`
@ -61,7 +61,7 @@ By default, Vagrant will share your project directory (the directory with the Va
Running the following commands in the guest machine should display these expected versions:
`node --version` should report *v0.12.7*
`node --version` should report *v0.12.7*
`mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85*
@ -104,7 +104,7 @@ Comment out the `forward_port` line, and uncomment the `private_network` line in
config.vm.network "private_network", ip: "192.168.51.52"
```
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
`vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`.
This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..

2
scripts/vagrant/zeppelin-dev/show-instructions.sh
View file

@ -18,7 +18,7 @@ echo '# Post vagrant up instructions.'
echo '# From your host machine,'
echo '# git clone the incubator-zeppelin branch into this directory'
echo
echo 'git clone https://github.com/apache/incubator-zeppelin.git'
echo 'git clone git://git.apache.org/incubator-zeppelin.git'
echo
echo '# Cloning the project again may seem counter intuitive, since this script'
echo '# originated from the project repository. Consider copying just the vagrant/zeppelin-dev'

7
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -66,14 +66,13 @@ public class ShellInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.debug("Run shell command '" + cmd + "'");
logger.error("user info found as :::" + contextInterpreter.getAuthenticationInfo().getUser());
CommandLine cmdLine = CommandLine.parse("bash");
cmdLine.addArgument("-c", false);
cmdLine.addArgument(cmd, false);
DefaultExecutor executor = new DefaultExecutor();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream));
executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeOut));
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
@ -83,7 +82,7 @@ public class ShellInterpreter extends Interpreter {
int exitVal = executor.execute(cmdLine);
logger.info("Paragraph " + contextInterpreter.getParagraphId()
+ "return with exit value: " + exitVal);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString());
return new InterpreterResult(InterpreterResult.Code.SUCCESS, null);
} catch (ExecuteException e) {
int exitValue = e.getExitValue();
logger.error("Can not run " + cmd, e);
@ -95,7 +94,7 @@ public class ShellInterpreter extends Interpreter {
logger.info("The paragraph " + contextInterpreter.getParagraphId()
+ " stopped executing: " + msg);
}
msg += "Exitvalue: " + exitValue;
msg += "ExitValue: " + exitValue;
return new InterpreterResult(code, msg);
} catch (IOException e) {
logger.error("Can not run " + cmd, e);

10
spark-dependencies/pom.xml
View file

@ -339,16 +339,6 @@
</dependencies>
<profiles>
<profile>
<id>vendor-repo</id>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
</profile>
<profile>
<id>spark-1.1</id>
<dependencies>

9
zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java
View file

@ -99,12 +99,17 @@ public class Message {
ANGULAR_OBJECT_UPDATE, // [s-c] add/update angular object
ANGULAR_OBJECT_REMOVE, // [s-c] add angular object del
ANGULAR_OBJECT_UPDATED, // [c-s] angular object value updated,
LIST_CONFIGURATIONS, // [c-s] ask all key/value pairs of configurations
CONFIGURATIONS_INFO // [s-c] all key/value pairs of configurations
CONFIGURATIONS_INFO, // [s-c] all key/value pairs of configurations
// @param settings serialized Map<String, String> object
CHECKPOINT_NOTEBOOK // [c-s] checkpoint notebook to storage repository
// @param noteId
// @param checkpointName
}
public OP op;

10
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -168,6 +168,9 @@ public class NotebookServer extends WebSocketServlet implements
case LIST_CONFIGURATIONS:
sendAllConfigurations(conn, notebook);
break;
case CHECKPOINT_NOTEBOOK:
checkpointNotebook(conn, notebook, messagereceived);
break;
default:
broadcastNoteList();
break;
@ -742,6 +745,13 @@ public class NotebookServer extends WebSocketServlet implements
.put("configurations", configurations)));
}
private void checkpointNotebook(NotebookSocket conn, Notebook notebook,
Message fromMessage) throws IOException {
String noteId = (String) fromMessage.get("noteId");
String commitMessage = (String) fromMessage.get("commitMessage");
notebook.checkpointNote(noteId, commitMessage);
}
/**
* This callback is for the paragraph that runs on ZeppelinServer
* @param noteId

2
zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
View file

@ -97,7 +97,7 @@ abstract public class AbstractZeppelinIT {
}
protected boolean endToEndTestEnabled() {
return null != System.getenv("CI");
return null != System.getenv("TEST_SELENIUM");
}
protected void createNewNote() {

12
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -58,7 +58,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
}
@Test
public void testMoveUpAndDown() throws InterruptedException {
public void testMoveUpAndDown() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
@ -111,14 +111,16 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
ZeppelinITUtils.sleep(1000,false);
deleteTestNotebook(driver);
} catch (ElementNotVisibleException e) {
} catch (Exception e) {
LOG.error("Exception in ParagraphActionsIT while testMoveUpAndDown ", e);
File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
throw e;
}
}
@Test
public void testDisableParagraphRunButton() throws InterruptedException {
public void testDisableParagraphRunButton() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
@ -149,8 +151,10 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
deleteTestNotebook(driver);
} catch (ElementNotVisibleException e) {
} catch (Exception e) {
LOG.error("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e);
File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
throw e;
}
}

97
zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java → zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java
View file

@ -15,8 +15,10 @@
* limitations under the License.
*/
package org.apache.zeppelin;
package org.apache.zeppelin.integration;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -35,8 +37,8 @@ import static org.junit.Assert.assertTrue;
* To test, ZeppelinServer should be running on port 8080
* On OSX, you'll need firefox 42.0 installed, then you can run with
*
* PATH=~/Applications/Firefox.app/Contents/MacOS/:$PATH CI="" \
* mvn -Dtest=org.apache.zeppelin.ZeppelinIT -Denforcer.skip=true \
* PATH=~/Applications/Firefox.app/Contents/MacOS/:$PATH TEST_SELENIUM="" \
* mvn -Dtest=org.apache.zeppelin.integration.ZeppelinIT -Denforcer.skip=true \
* test -pl zeppelin-server
*
*/
@ -62,7 +64,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
}
@Test
public void testAngularDisplay() throws InterruptedException{
public void testAngularDisplay() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
@ -193,55 +195,68 @@ public class ZeppelinIT extends AbstractZeppelinIT {
sleep(100, true);
System.out.println("testCreateNotebook Test executed");
} catch (ElementNotVisibleException e) {
} catch (Exception e) {
LOG.error("Exception in ZeppelinIT while testAngularDisplay ", e);
File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
throw e;
}
}
@Test
public void testSparkInterpreterDependencyLoading() {
// navigate to interpreter page
WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
interpreterLink.click();
public void testSparkInterpreterDependencyLoading() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// navigate to interpreter page
WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
interpreterLink.click();
// add new dependency to spark interpreter
WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
String artifact = "org.apache.commons:commons-csv:1.1";
depArtifact.sendKeys(artifact);
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.switchTo().alert().accept();
// add new dependency to spark interpreter
WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
String artifact = "org.apache.commons:commons-csv:1.1";
depArtifact.sendKeys(artifact);
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.switchTo().alert().accept();
driver.navigate().back();
createNewNote();
driver.navigate().back();
createNewNote();
// wait for first paragraph's " READY " status text
waitForParagraph(1, "READY");
// wait for first paragraph's " READY " status text
waitForParagraph(1, "READY");
WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
paragraph1Editor.sendKeys("import org.apache.commons.csv.CSVFormat");
paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
waitForParagraph(1, "FINISHED");
paragraph1Editor.sendKeys("import org.apache.commons.csv.CSVFormat");
paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
waitForParagraph(1, "FINISHED");
// check expected text
assertTrue(waitForText("import org.apache.commons.csv.CSVFormat",
By.xpath(getParagraphXPath(1) + "//div[starts-with(@id, 'p') and contains(@id, 'text')]/div")));
// check expected text
assertTrue(waitForText("import org.apache.commons.csv.CSVFormat",
By.xpath(getParagraphXPath(1) + "//div[starts-with(@id, 'p') and contains(@id, 'text')]/div")));
// reset dependency
interpreterLink.click();
sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"));
sleep(5000, true);
testDepRemoveBtn.click();
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.switchTo().alert().accept();
//delete created notebook for cleanup.
deleteTestNotebook(driver);
sleep(1000, true);
// reset dependency
interpreterLink.click();
sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
MAX_BROWSER_TIMEOUT_SEC);
sparkEditBtn.click();
WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
artifact + "')]]/td[3]/div"));
sleep(5000, true);
testDepRemoveBtn.click();
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
driver.switchTo().alert().accept();
} catch (Exception e) {
LOG.error("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
throw e;
}
}
}

25
zeppelin-web/src/app/home/home.css
View file

@ -126,6 +126,31 @@ a.navbar-brand:hover {
outline: 0;
}
#notebook-list > .filter-names {
margin: 5px;
padding: 0px 10px;
}
#notebook-list .note-name-query {
width: 100%;
}
#notebook-names {
list-style: none;
}
#notebook-names > .filter-names {
margin: 5px 0;
}
.note-name-query {
padding: 6px;
color: #000;
height: 28px;
width: 200px;
font: normal normal normal 14px/1 FontAwesome;
}
@media (max-width: 767px) {
.navbar-inverse .navbar-nav .open .dropdown-menu > li > a {
color: #D3D3D3;

7
zeppelin-web/src/app/home/home.html
View file

@ -26,7 +26,7 @@ limitations under the License.
<div class="row">
<div class="col-md-4">
<h4>Notebook
<h4>Notebook
<i ng-class="isReloadingNotes ? 'fa fa-refresh fa-spin' : 'fa fa-refresh'"
ng-style="!isReloadingNotes && {'cursor': 'pointer'}" style="font-size: 13px;"
ng-click="reloadNotebookList();">
@ -38,8 +38,9 @@ limitations under the License.
<i style="font-size: 15px;" class="fa fa-upload"></i> Import note</a></h5>
<h5><a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;">
<i style="font-size: 15px;" class="icon-notebook"></i> Create new note</a></h5>
<ul style="list-style-type: none;">
<li ng-repeat="note in home.notes.list | orderBy:home.arrayOrderingSrv.notebookListOrdering track by $index">
<ul id="notebook-names">
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
<li ng-repeat="note in home.notes.list | filter:query | orderBy:home.arrayOrderingSrv.notebookListOrdering track by $index">
<i style="font-size: 10px;" class="icon-doc"></i>
<a style="text-decoration: none;" href="#/notebook/{{note.id}}">{{note.name || 'Note ' + note.id}}</a>
</li>

26
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -61,6 +61,32 @@ limitations under the License.
tooltip-placement="bottom" tooltip="Export the notebook">
<i class="fa fa-download"></i>
</button>
<ul class="dropdown-menu" role="menu" style="width:250px">
<li>
<div class="cron-preset-container">
<div>
<input type="text"
dropdown-input
placeholder="commit message"
id="note.checkpoint.message"
ng-model="note.checkpoint.message"/>
<button type="button"
class="btn btn-default btn-xs"
ng-hide="viewOnly"
ng-click="checkpointNotebook(note.checkpoint.message)"
tooltip-placement="bottom" tooltip="Commit the notebook">Commit
</button>
</div>
</div>
</li>
</ul>
<button type="button"
class="btn btn-default btn-xs dropdown-toggle"
ng-hide="viewOnly"
data-toggle="dropdown"
tooltip-placement="bottom" tooltip="Version control">
<i class="fa fa-file-code-o"></i>
</button>
</span>
<!-- put the delete action by itself for your protection -->

15
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -154,6 +154,21 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
});
};
// checkpoint/commit notebook
$scope.checkpointNotebook = function(commitMessage) {
BootstrapDialog.confirm({
closable: true,
title: '',
message: 'Commit notebook to current repository?',
callback: function(result) {
if (result) {
websocketMsgSrv.checkpointNotebook($routeParams.noteId, commitMessage);
}
}
});
document.getElementById('note.checkpoint.message').value='';
};
$scope.runNote = function() {
BootstrapDialog.confirm({
closable: true,

17
zeppelin-web/src/app/notebook/paragraph/paragraph-results.html
View file

@ -23,9 +23,20 @@ limitations under the License.
ng-bind-html="paragraph.result.comment">
</div>
<div id="p{{paragraph.id}}_text"
class="text"
ng-if="getResultType() == 'TEXT'"></div>
<div id="{{paragraph.id}}_text"
ng-if="getResultType() == 'TEXT'">
<div class="fa fa-level-down scroll-paragraph-down"
ng-show="showScrollDownIcon()"
ng-click="scrollParagraphDown()"
tooltip="Follow Output"></div>
<div id="p{{paragraph.id}}_text"
style="max-height: {{paragraph.config.graph.height}}px; overflow: auto"
class="text"></div>
<div class="fa fa-chevron-up scroll-paragraph-up"
ng-show="showScrollUpIcon()"
ng-click="scrollParagraphUp()"
tooltip="Scroll Top"></div>
</div>
<div id="p{{paragraph.id}}_html"
class="resultContained"

44
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -108,6 +108,11 @@ angular.module('zeppelinWebApp')
if ($scope.paragraph.result && $scope.paragraph.result.msg) {
$scope.appendTextOutput($scope.paragraph.result.msg);
}
angular.element('#p' + $scope.paragraph.id + '_text').bind("mousewheel", function(e) {
$scope.keepScrollDown = false;
});
} else {
$timeout(retryRenderer, 10);
}
@ -130,6 +135,10 @@ angular.module('zeppelinWebApp')
textEl.append(angular.element('<div></div>').text(lines[i]));
}
}
if ($scope.keepScrollDown) {
var doc = angular.element('#p' + $scope.paragraph.id + '_text');
doc[0].scrollTop = doc[0].scrollHeight;
}
};
@ -1238,7 +1247,9 @@ angular.module('zeppelinWebApp')
};
var integerFormatter = d3.format(',.1d');
var groupedThousandsWith3DigitsFormatter = function(x){
return d3.format(',')(d3.round(x, 3));
};
var customAbbrevFormatter = function(x) {
var s = d3.format('.3s')(x);
@ -1260,7 +1271,7 @@ angular.module('zeppelinWebApp')
if(d >= Math.pow(10,6)){
return customAbbrevFormatter(d);
}
return integerFormatter(d);
return groupedThousandsWith3DigitsFormatter(d);
};
var setD3Chart = function(type, data, refresh) {
@ -2075,4 +2086,33 @@ angular.module('zeppelinWebApp')
var redirectToUrl = location.protocol + '//' + location.host + location.pathname + '#/notebook/' + noteId + '/paragraph/' + $scope.paragraph.id+'?asIframe';
$window.open(redirectToUrl);
};
$scope.showScrollDownIcon = function(){
var doc = angular.element('#p' + $scope.paragraph.id + '_text');
if(doc[0]){
return doc[0].scrollHeight > doc.innerHeight();
}
return false;
};
$scope.scrollParagraphDown = function() {
var doc = angular.element('#p' + $scope.paragraph.id + '_text');
doc.animate({scrollTop: doc[0].scrollHeight}, 500);
$scope.keepScrollDown = true;
};
$scope.showScrollUpIcon = function(){
if(angular.element('#p' + $scope.paragraph.id + '_text')[0]){
return angular.element('#p' + $scope.paragraph.id + '_text')[0].scrollTop != 0;
}
return false;
};
$scope.scrollParagraphUp = function() {
var doc = angular.element('#p' + $scope.paragraph.id + '_text');
doc.animate({scrollTop: 0}, 500);
$scope.keepScrollDown = false;
};
});

14
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -411,3 +411,17 @@ table.table-striped {
border-top: 1px solid #ddd;
margin-top: 20px;
}
.scroll-paragraph-down {
position: absolute;
right: 10px;
cursor: pointer;
}
.scroll-paragraph-up {
bottom: 5px;
cursor: pointer;
position: absolute;
right: 15px;
}

14
zeppelin-web/src/components/filterNoteNames/filter-note-names.html Normal file
View file

@ -0,0 +1,14 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<input type="text" class="note-name-query form-control" ng-click="$event.stopPropagation()" placeholder="&#xf002 Filter" ng-model="$parent.query.name" />

5
zeppelin-web/src/components/navbar/navbar.html
View file

@ -20,7 +20,7 @@ limitations under the License.
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="#/">
<img style="margin-top: -7px;s" src="assets/images/zepLogoW.png" width="50" alt="I'm zeppelin"> Zeppelin
<img style="margin-top: -7px;" src="assets/images/zepLogoW.png" width="50" alt="I'm zeppelin"> Zeppelin
</a>
</div>
@ -32,7 +32,8 @@ limitations under the License.
<li><a href="" data-toggle="modal" data-target="#noteNameModal"><i class="fa fa-plus"></i> Create new note</a></li>
<li class="divider"></li>
<div id="notebook-list" class="scrollbar-container">
<li ng-repeat="note in navbar.notes.list | orderBy:navbar.arrayOrderingSrv.notebookListOrdering track by $index"
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
<li ng-repeat="note in navbar.notes.list | filter:query | orderBy:navbar.arrayOrderingSrv.notebookListOrdering track by $index"
ng-class="{'active' : navbar.isActive(note.id)}">
<a href="#/notebook/{{note.id}}">{{note.name || 'Note ' + note.id}} </a>
</li>

2
zeppelin-web/src/components/resizable/resizable.directive.js
View file

@ -35,7 +35,7 @@ angular.module('zeppelinWebApp').directive('resizable', function() {
var colStep = window.innerWidth / 12;
elem.off('resizestop');
var conf = angular.copy(resizableConfig);
if (resize.graphType === 'TABLE') {
if (resize.graphType === 'TABLE' || resize.graphType === 'TEXT') {
conf.grid = [colStep, 10];
conf.minHeight = 100;
} else {

10
zeppelin-web/src/components/websocketEvents/websocketMsg.service.js
View file

@ -128,6 +128,16 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope,
});
},
checkpointNotebook: function(noteId, commitMessage) {
websocketEvents.sendNewEvent({
op: 'CHECKPOINT_NOTEBOOK',
data: {
noteId: noteId,
commitMessage: commitMessage
}
});
},
isConnected: function(){
return websocketEvents.isConnected();
}

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -307,6 +307,10 @@ public class Notebook {
}
}
public void checkpointNote(String noteId, String checkpointMessage) throws IOException {
notebookRepo.checkpoint(noteId, checkpointMessage);
}
@SuppressWarnings("rawtypes")
private Note loadNoteFromRepo(String id) {
Note note = null;

21
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
View file

@ -25,6 +25,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.dircache.DirCache;
import org.eclipse.jgit.internal.storage.file.FileRepository;
@ -61,28 +62,33 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoVers
localRepo.create();
}
git = new Git(localRepo);
maybeAddAndCommit(".");
}
@Override
public synchronized void save(Note note) throws IOException {
super.save(note);
maybeAddAndCommit(note.getId());
}
private void maybeAddAndCommit(String pattern) {
/* implemented as git add+commit
* @param pattern is the noteId
* @param commitMessage is a commit message (checkpoint name)
* (non-Javadoc)
* @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String)
*/
@Override
public void checkpoint(String pattern, String commitMessage) {
try {
List<DiffEntry> gitDiff = git.diff().call();
if (!gitDiff.isEmpty()) {
LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff);
DirCache added = git.add().addFilepattern(pattern).call();
LOG.debug("{} changes are about to be commited", added.getEntryCount());
git.commit().setMessage("Updated " + pattern).call();
git.commit().setMessage(commitMessage).call();
} else {
LOG.debug("No changes found {}", pattern);
}
} catch (GitAPIException e) {
LOG.error("Faild to add+comit {} to Git", pattern, e);
LOG.error("Failed to add+comit {} to Git", pattern, e);
}
}
@ -100,8 +106,11 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoVers
Iterable<RevCommit> logs = git.log().addPath(noteId).call();
for (RevCommit log: logs) {
history.add(new Rev(log.getName(), log.getCommitTime()));
LOG.debug(" - ({},{})", log.getName(), log.getCommitTime());
LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage());
}
} catch (NoHeadException e) {
//when no initial commit exists
LOG.warn("No Head found for {}, {}", noteId, e.getMessage());
} catch (GitAPIException e) {
LOG.error("Failed to get logs for {}", noteId, e);
}

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepo.java
View file

@ -36,4 +36,9 @@ public interface NotebookRepo {
* Release any underlying resources
*/
public void close();
/**
* chekpoint (versioning) for notebooks (optional)
*/
public void checkpoint(String noteId, String checkPointName) throws IOException;
}

26
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
View file

@ -224,7 +224,8 @@ public class NotebookRepoSync implements NotebookRepo {
NotebookRepo getRepo(int repoIndex) throws IOException {
if (repoIndex < 0 || repoIndex >= getRepoCount()) {
throw new IOException("Storage repo index is out of range");
throw new IOException("Requested storage index " + repoIndex
+ " isn't initialized," + " repository count is " + getRepoCount());
}
return repos.get(repoIndex);
}
@ -351,4 +352,27 @@ public class NotebookRepoSync implements NotebookRepo {
}
}
//checkpoint to all available storages
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
int repoCount = getRepoCount();
int errorCount = 0;
String errorMessage = "";
for (int i = 0; i < Math.min(repoCount, getMaxRepoNum()); i++) {
try {
getRepo(i).checkpoint(noteId, checkPointName);
} catch (IOException e) {
LOG.warn("Couldn't checkpoint in {} storage with index {} for note {}",
getRepo(i).getClass().toString(), i, noteId);
errorMessage += "Error on storage class " + getRepo(i).getClass().toString() +
" with index " + i + " : " + e.getMessage() + "\n";
errorCount++;
}
}
// throw exception if failed to commit for all initialized repos
if (errorCount == Math.min(repoCount, getMaxRepoNum())) {
throw new IOException(errorMessage);
}
}
}

6
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -189,4 +189,10 @@ public class S3NotebookRepo implements NotebookRepo {
public void close() {
//no-op
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
// no-op
LOG.info("Checkpoint feature isn't suported in {}", this.getClass().toString());
}
}

6
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
View file

@ -244,4 +244,10 @@ public class VFSNotebookRepo implements NotebookRepo {
//no-op
}
@Override
public void checkpoint(String noteId, String checkPointName) throws IOException {
// no-op
logger.info("Checkpoint feature isn't suported in {}", this.getClass().toString());
}
}

48
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
View file

@ -22,12 +22,16 @@ import static com.google.common.truth.Truth.assertThat;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.repo.NotebookRepoVersioned.Rev;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
@ -96,7 +100,8 @@ public class GitNotebookRepoTest {
assertThat(notebookRepo.list()).isNotEmpty();
List<DiffEntry> diff = git.diff().call();
assertThat(diff).isEmpty();
// no commit, diff isn't empty
assertThat(diff).isNotEmpty();
}
@Test
@ -109,7 +114,46 @@ public class GitNotebookRepoTest {
List<Rev> testNotebookHistory = notebookRepo.history(TEST_NOTE_ID);
//then
assertThat(testNotebookHistory).isNotEmpty();
//no initial commit, empty history
assertThat(testNotebookHistory).isEmpty();
}
@Test
public void addCheckpoint() throws IOException {
// initial checks
notebookRepo = new GitNotebookRepo(conf);
assertThat(notebookRepo.list()).isNotEmpty();
assertThat(containsNote(notebookRepo.list(), TEST_NOTE_ID)).isTrue();
assertThat(notebookRepo.history(TEST_NOTE_ID)).isEmpty();
notebookRepo.checkpoint(TEST_NOTE_ID, "first commit");
List<Rev> notebookHistoryBefore = notebookRepo.history(TEST_NOTE_ID);
assertThat(notebookRepo.history(TEST_NOTE_ID)).isNotEmpty();
int initialCount = notebookHistoryBefore.size();
// add changes to note
Note note = notebookRepo.get(TEST_NOTE_ID);
Paragraph p = note.addParagraph();
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test text");
// save and checkpoint note
notebookRepo.save(note);
notebookRepo.checkpoint(TEST_NOTE_ID, "second commit");
// see if commit is added
List<Rev> notebookHistoryAfter = notebookRepo.history(TEST_NOTE_ID);
assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1);
}
private boolean containsNote(List<NoteInfo> notes, String noteId) {
for (NoteInfo note: notes) {
if (note.getId().equals(noteId)) {
return true;
}
}
return false;
}
}

43
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.notebook.repo;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@ -44,6 +45,7 @@ import org.apache.zeppelin.search.LuceneSearch;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,6 +61,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private NotebookRepoSync notebookRepoSync;
private InterpreterFactory factory;
private DependencyResolver depResolver;
private SearchService search;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
@Before
@ -90,7 +93,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver);
SearchService search = mock(SearchService.class);
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search);
}
@ -211,6 +214,44 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, notebookRepoSync.list(1).size());
}
@Test
public void testCheckpointOneStorage() throws IOException, SchedulerException {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo");
ZeppelinConfiguration vConf = ZeppelinConfiguration.create();
NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf);
Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, this, search);
// one git versioned storage initialized
assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
// no notes
assertThat(vRepoSync.list().size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote();
assertThat(vRepoSync.list().size()).isEqualTo(1);
String noteId = vRepoSync.list().get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message");
int vCount = gitRepo.history(noteId).size();
assertThat(vCount).isEqualTo(1);
Paragraph p = note.addParagraph();
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test");
// save and checkpoint again
vRepoSync.save(note);
vRepoSync.checkpoint(noteId, "checkpoint message 2");
assertThat(gitRepo.history(noteId).size()).isEqualTo(vCount + 1);
}
static void delete(File file){
if(file.isFile()) file.delete();
else if(file.isDirectory()){