mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge branch 'master' into ZEPPELIN-732-up
This commit is contained in:
Lee moon soo
commit
ab1de033d9
71 changed files with 889 additions and 587 deletions
|
|
@ -87,4 +87,4 @@ if [[ ! -d "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
|
|||
$(mkdir -p "${ZEPPELIN_NOTEBOOK_DIR}")
|
||||
fi
|
||||
|
||||
$(exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_SERVER "$@")
|
||||
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_SERVER "$@"
|
||||
|
|
|
|||
|
|
@ -302,10 +302,10 @@ public class InterpreterLogicTest {
|
|||
}
|
||||
|
||||
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
|
||||
return scala.collection.JavaConversions.asScalaIterable(list).toList();
|
||||
return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList();
|
||||
}
|
||||
|
||||
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
|
||||
return scala.collection.JavaConversions.asJavaList(list);
|
||||
return scala.collection.JavaConversions.seqAsJavaList(list);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# This is the default format.
|
||||
# This is the default format.
|
||||
# For more see: http://jekyllrb.com/docs/permalinks/
|
||||
permalink: /:categories/:year/:month/:day/:title
|
||||
permalink: /:categories/:year/:month/:day/:title
|
||||
|
||||
exclude: [".rvmrc", ".rbenv-version", "README.md", "Rakefile", "changelog.md", "vendor", "node_modules", "scss"]
|
||||
pygments: true
|
||||
|
|
@ -9,7 +9,7 @@ redcarpet:
|
|||
extensions: ["tables"]
|
||||
encoding: utf-8
|
||||
|
||||
# Themes are encouraged to use these universal variables
|
||||
# Themes are encouraged to use these universal variables
|
||||
# so be sure to set them if your theme uses them.
|
||||
#
|
||||
title : Apache Zeppelin (incubating)
|
||||
|
|
@ -24,7 +24,7 @@ author :
|
|||
ZEPPELIN_VERSION : 0.6.0-incubating-SNAPSHOT
|
||||
|
||||
# The production_url is only used when full-domain names are needed
|
||||
# such as sitemap.txt
|
||||
# such as sitemap.txt
|
||||
# Most places will/should use BASE_PATH to make the urls
|
||||
#
|
||||
# If you have set a CNAME (pages.github.com) set your custom domain here.
|
||||
|
|
@ -42,11 +42,11 @@ JB :
|
|||
# however this value will be dynamically changed depending on your deployment situation.
|
||||
#
|
||||
# CNAME (http://yourcustomdomain.com)
|
||||
# DO NOT SET BASE_PATH
|
||||
# DO NOT SET BASE_PATH
|
||||
# (urls will be prefixed with "/" and work relatively)
|
||||
#
|
||||
# GitHub Pages (http://username.github.io)
|
||||
# DO NOT SET BASE_PATH
|
||||
# DO NOT SET BASE_PATH
|
||||
# (urls will be prefixed with "/" and work relatively)
|
||||
#
|
||||
# GitHub Project Pages (http://username.github.io/project-name)
|
||||
|
|
@ -65,7 +65,7 @@ JB :
|
|||
# ex: [BASE_PATH]/assets/themes/[THEME-NAME]
|
||||
#
|
||||
# Override this by defining an absolute path to assets here.
|
||||
# ex:
|
||||
# ex:
|
||||
# http://s3.amazonaws.com/yoursite/themes/watermelon
|
||||
# /assets
|
||||
#
|
||||
|
|
@ -97,42 +97,41 @@ JB :
|
|||
num_posts: 5
|
||||
width: 580
|
||||
colorscheme: light
|
||||
|
||||
|
||||
# Settings for analytics helper
|
||||
# Set 'provider' to the analytics provider you want to use.
|
||||
# Set 'provider' to false to turn analytics off globally.
|
||||
#
|
||||
#
|
||||
analytics :
|
||||
provider : google_universal
|
||||
google_classic :
|
||||
google_classic :
|
||||
tracking_id : 'UA-45176241-2'
|
||||
google_universal :
|
||||
google_universal :
|
||||
tracking_id : 'UA-45176241-5'
|
||||
domain : 'zeppelin.incubator.apache.org'
|
||||
getclicky :
|
||||
site_id :
|
||||
site_id :
|
||||
mixpanel :
|
||||
token : '_MIXPANEL_TOKEN_'
|
||||
piwik :
|
||||
baseURL : 'myserver.tld/piwik' # Piwik installation address (without protocol)
|
||||
idsite : '1' # the id of the site on Piwik
|
||||
|
||||
# Settings for sharing helper.
|
||||
# Settings for sharing helper.
|
||||
# Sharing is for things like tweet, plusone, like, reddit buttons etc.
|
||||
# Set 'provider' to the sharing provider you want to use.
|
||||
# Set 'provider' to false to turn sharing off globally.
|
||||
#
|
||||
sharing :
|
||||
provider : false
|
||||
|
||||
# Settings for all other include helpers can be defined by creating
|
||||
|
||||
# Settings for all other include helpers can be defined by creating
|
||||
# a hash with key named for the given helper. ex:
|
||||
#
|
||||
# pages_list :
|
||||
# provider : "custom"
|
||||
# provider : "custom"
|
||||
#
|
||||
# Setting any helper's provider to 'custom' will bypass the helper code
|
||||
# and include your custom code. Your custom file must be defined at:
|
||||
# ./_includes/custom/[HELPER]
|
||||
# where [HELPER] is the name of the helper you are overriding.
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 176 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 42 KiB |
|
|
@ -4,7 +4,7 @@ title : Atom Feed
|
|||
---
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
|
||||
|
||||
<title>{{ site.title }}</title>
|
||||
<link href="{{ site.production_url }}/{{ site.atom_path }}" rel="self"/>
|
||||
<link href="{{ site.production_url }}"/>
|
||||
|
|
@ -24,5 +24,5 @@ title : Atom Feed
|
|||
<content type="html">{{ post.content | xml_escape }}</content>
|
||||
</entry>
|
||||
{% endfor %}
|
||||
|
||||
</feed>
|
||||
|
||||
</feed>
|
||||
|
|
|
|||
|
|
@ -12,20 +12,20 @@ Apache Zeppelin (incubating) is an [Apache2 License](http://www.apache.org/licen
|
|||
Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License.
|
||||
|
||||
## Setting up
|
||||
Here are some tools you will need to build and test Zeppelin.
|
||||
Here are some tools you will need to build and test Zeppelin.
|
||||
|
||||
#### Software Configuration Management ( SCM )
|
||||
|
||||
Since Zeppelin uses Git for it's SCM system, you need git client installed in your development machine.
|
||||
Since Zeppelin uses Git for it's SCM system, you need git client installed in your development machine.
|
||||
|
||||
#### Integrated Development Environment ( IDE )
|
||||
|
||||
You are free to use whatever IDE you prefer, or your favorite command line editor.
|
||||
You are free to use whatever IDE you prefer, or your favorite command line editor.
|
||||
|
||||
### Build Tools
|
||||
|
||||
To build the code, install
|
||||
|
||||
|
||||
* Oracle Java 7
|
||||
* Apache Maven
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ limitations under the License.
|
|||
### What is Zeppelin Interpreter
|
||||
|
||||
Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need scala interpreter.
|
||||
Every Interpreter belongs to an InterpreterGroup.
|
||||
Every Interpreter belongs to an InterpreterGroup.
|
||||
Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group.
|
||||
|
||||
<img class="img-responsive" style="width:50%; border: 1px solid #ecf0f1;" height="auto" src="/assets/themes/zeppelin/img/interpreter.png" />
|
||||
|
|
|
|||
|
|
@ -92,12 +92,12 @@ When the button is clicked, you'll see both `run` and `numWatched` are increment
|
|||
<img src="/assets/themes/zeppelin/img/screenshots/display_angular3.png" width="60%" />
|
||||
|
||||
## Let's make it Simpler and more Intuitive
|
||||
In this section, we will introduce a simpler and more intuitive way of using **Angular Display System** in Zeppelin.
|
||||
In this section, we will introduce a simpler and more intuitive way of using **Angular Display System** in Zeppelin.
|
||||
|
||||
### How can we use it?
|
||||
Here are some usages.
|
||||
Here are some usages.
|
||||
|
||||
#### Import
|
||||
#### Import
|
||||
|
||||
##### - In notebook scope
|
||||
```scala
|
||||
|
|
@ -141,11 +141,11 @@ import AngularElem._
|
|||
<div></div>.model("myModel").display
|
||||
|
||||
// bind model with initial value
|
||||
<div></div>.model("myModel", initialValue).display
|
||||
<div></div>.model("myModel", initialValue).display
|
||||
```
|
||||
|
||||
#### Interact with Model
|
||||
```scala
|
||||
```scala
|
||||
// read model
|
||||
AngularModel("myModel")()
|
||||
|
||||
|
|
@ -155,7 +155,7 @@ AngularModel("myModel", "newValue")
|
|||
|
||||
<br/>
|
||||
### Example: Basic Usage
|
||||
Using the above basic usages, you can apply them like below examples.
|
||||
Using the above basic usages, you can apply them like below examples.
|
||||
|
||||
#### Display Elements
|
||||
|
||||
|
|
@ -195,7 +195,7 @@ AngularModel("myModel", "New value")
|
|||
|
||||
### Example: String Converter
|
||||
Using below example, you can convert the lowercase string to uppercase.
|
||||
|
||||
|
||||
{% raw %}
|
||||
```scala
|
||||
// clear previously created angular object.
|
||||
|
|
@ -215,5 +215,3 @@ val button = <div class="btn btn-success btn-sm">Convert</div>.onClick{() =>
|
|||
{% endraw %}
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/string-converter-angular.gif" width="70%">
|
||||
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ Some basic charts are already included in Zeppelin. Visualizations are not limit
|
|||
|
||||
#### Pivot chart
|
||||
|
||||
With simple drag and drop Zeppelin aggeregates the values and display them in pivot chart. You can easily create chart with multiple aggregated values including sum, count, average, min, max.
|
||||
With simple drag and drop Zeppelin aggregates the values and display them in pivot chart. You can easily create chart with multiple aggregated values including sum, count, average, min, max.
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8">
|
||||
|
|
@ -123,4 +123,4 @@ Join the [Mailing list](./community.html) and report issues on our [Issue tracke
|
|||
|
||||
<br />
|
||||
### Undergoing Incubation
|
||||
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
|
||||
Apache Zeppelin is an effort undergoing [incubation](https://incubator.apache.org/index.html) at The Apache Software Foundation (ASF), sponsored by the Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
|
||||
|
|
|
|||
|
|
@ -295,4 +295,3 @@ exec bin/zeppelin-daemon.sh upstart
|
|||
```
|
||||
bin\zeppelin.cmd
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -41,4 +41,4 @@ So, copying `notebook` and `conf` directory should be enough.
|
|||
|
||||
```
|
||||
bin/zeppelin-daemon.sh start
|
||||
```
|
||||
```
|
||||
|
|
|
|||
|
|
@ -21,15 +21,15 @@ limitations under the License.
|
|||
|
||||
|
||||
## Vagrant Virtual Machine for Apache Zeppelin
|
||||
|
||||
|
||||
Apache Zeppelin distribution includes a scripts directory
|
||||
|
||||
`scripts/vagrant/zeppelin-dev`
|
||||
|
||||
|
||||
This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin. It can also be used to run an existing Zeppelin build if you don't plan to build from source.
|
||||
For PySpark users, this script includes several helpful [Python Libraries](#python-extras).
|
||||
For SparkR users, this script includes several helpful [R Libraries](#r-extras).
|
||||
|
||||
|
||||
####Installing the required components to launch a virtual machine.
|
||||
|
||||
This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box"). All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
|
||||
|
|
@ -40,11 +40,11 @@ If you are running Windows and don't yet have python installed, [install Python
|
|||
|
||||
1. Download and Install Vagrant: [Vagrant Downloads](http://www.vagrantup.com/downloads)
|
||||
2. Install Ansible: [Ansible Python pip install](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip)
|
||||
|
||||
|
||||
```
|
||||
sudo easy_install pip
|
||||
sudo pip install ansible
|
||||
ansible --version
|
||||
ansible --version
|
||||
```
|
||||
After then, please check whether it reports **ansible version 1.9.2 or higher**.
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ Cloning the project again may seem counter intuitive, since this script likley o
|
|||
|
||||
Synced folders enable Vagrant to sync a folder on the host machine to the guest machine, allowing you to continue working on your project's files on your host machine, but use the resources in the guest machine to compile or run your project. _[(1) Synced Folder Description from Vagrant Up](https://docs.vagrantup.com/v2/synced-folders/index.html)_
|
||||
|
||||
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
|
||||
By default, Vagrant will share your project directory (the directory with the Vagrantfile) to `/vagrant`. Which means you should be able to build within the guest machine after you
|
||||
`cd /vagrant/incubator-zeppelin`
|
||||
|
||||
|
||||
|
|
@ -95,7 +95,7 @@ The virtual machine consists of:
|
|||
- openjdk-7-jdk
|
||||
- Python addons: pip, matplotlib, scipy, numpy, pandas
|
||||
- [R](https://www.r-project.org/) and R Packages required to run the R Interpreter and the related R tutorial notebook, including: Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table
|
||||
|
||||
|
||||
### How to build & run Zeppelin
|
||||
|
||||
This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine. The following build steps will also include Python and R support via PySpark and SparkR:
|
||||
|
|
@ -138,7 +138,7 @@ import scipy
|
|||
import pandas
|
||||
import matplotlib
|
||||
|
||||
print "numpy " + numpy.__version__
|
||||
print "numpy " + numpy.__version__
|
||||
print "scipy " + scipy.__version__
|
||||
print "pandas " + pandas.__version__
|
||||
print "matplotlib " + matplotlib.__version__
|
||||
|
|
@ -176,7 +176,7 @@ plt.xlabel('Performance')
|
|||
plt.title('How fast do you want to go today?')
|
||||
|
||||
show(plt)
|
||||
```
|
||||
```
|
||||
|
||||
### R Extras
|
||||
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@ This page describes how to pre-configure a bare metal node, configure Zeppelin a
|
|||
## Prepare Node
|
||||
|
||||
### Zeppelin user (Optional)
|
||||
This step is optional, however its nice to run Zeppelin under its own user. In case you do not like to use Zeppelin (hope not) the user could be deleted along with all the pacakges that were installed for Zeppelin, Zeppelin binary itself and associated directories.
|
||||
This step is optional, however its nice to run Zeppelin under its own user. In case you do not like to use Zeppelin (hope not) the user could be deleted along with all the packages that were installed for Zeppelin, Zeppelin binary itself and associated directories.
|
||||
|
||||
Create a zeppelin user and switch to zeppelin user or if zeppelin user is already created then login as zeppelin.
|
||||
|
||||
```bash
|
||||
useradd zeppelin
|
||||
su - zeppelin
|
||||
su - zeppelin
|
||||
whoami
|
||||
```
|
||||
Assuming a zeppelin user is created then running whoami command must return
|
||||
Assuming a zeppelin user is created then running whoami command must return
|
||||
|
||||
```bash
|
||||
zeppelin
|
||||
|
|
@ -48,7 +48,7 @@ Its assumed in the rest of the document that zeppelin user is indeed created and
|
|||
* Java 1.7
|
||||
* Hadoop client
|
||||
* Spark
|
||||
* Internet connection is required.
|
||||
* Internet connection is required.
|
||||
|
||||
It's assumed that the node has CentOS 6.x installed on it. Although any version of Linux distribution should work fine.
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ This document assumes that Zeppelin is located under `/home/zeppelin/incubator-z
|
|||
Zeppelin configuration needs to be modified to connect to YARN cluster. Create a copy of zeppelin environment shell script.
|
||||
|
||||
```bash
|
||||
cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh
|
||||
cp /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh.template /home/zeppelin/incubator-zeppelin/conf/zeppelin-env.sh
|
||||
```
|
||||
|
||||
Set the following properties
|
||||
|
|
@ -127,7 +127,7 @@ Zeppelin supports Hive interpreter and hence copy hive-site.xml that should be p
|
|||
cp /etc/hive/conf/hive-site.xml /home/zeppelin/incubator-zeppelin/conf
|
||||
```
|
||||
|
||||
Once Zeppelin server has started successfully, visit http://[zeppelin-server-host-name]:8080 with your web browser. Click on Interpreter tab next to Notebook dropdown. Look for Hive configurations and set them appropriately. By default hive.hiveserver2.url will be pointing to localhost and hive.hiveserver2.password/hive.hiveserver2.user are set to hive/hive. Set them as per Hive installation on YARN cluster.
|
||||
Once Zeppelin server has started successfully, visit http://[zeppelin-server-host-name]:8080 with your web browser. Click on Interpreter tab next to Notebook dropdown. Look for Hive configurations and set them appropriately. By default hive.hiveserver2.url will be pointing to localhost and hive.hiveserver2.password/hive.hiveserver2.user are set to hive/hive. Set them as per Hive installation on YARN cluster.
|
||||
Click on Save button. Once these configurations are updated, Zeppelin will prompt you to restart the interpreter. Accept the prompt and the interpreter will reload the configurations.
|
||||
|
||||
### Spark
|
||||
|
|
@ -161,7 +161,7 @@ Click on Save button. Once these configurations are updated, Zeppelin will promp
|
|||
Spark & Hive notebooks can be written with Zeppelin now. The resulting Spark & Hive jobs will run on configured YARN cluster.
|
||||
|
||||
## Debug
|
||||
Zeppelin does not emit any kind of error messages on web interface when notebook/paragrah is run. If a paragraph fails it only displays ERROR. The reason for failure needs to be looked into log files which is present in logs directory under zeppelin installation base directory. Zeppelin creates a log file for each kind of interpreter.
|
||||
Zeppelin does not emit any kind of error messages on web interface when notebook/paragraph is run. If a paragraph fails it only displays ERROR. The reason for failure needs to be looked into log files which is present in logs directory under zeppelin installation base directory. Zeppelin creates a log file for each kind of interpreter.
|
||||
|
||||
```bash
|
||||
[zeppelin@zeppelin-3529 logs]$ pwd
|
||||
|
|
@ -172,5 +172,5 @@ total 844
|
|||
-rw-rw-r-- 1 zeppelin zeppelin 625050 Aug 3 16:05 zeppelin-interpreter-spark-zeppelin-zeppelin-3529.log
|
||||
-rw-rw-r-- 1 zeppelin zeppelin 200394 Aug 3 21:15 zeppelin-zeppelin-zeppelin-3529.log
|
||||
-rw-rw-r-- 1 zeppelin zeppelin 16162 Aug 3 14:03 zeppelin-zeppelin-zeppelin-3529.out
|
||||
[zeppelin@zeppelin-3529 logs]$
|
||||
[zeppelin@zeppelin-3529 logs]$
|
||||
```
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ The **Alluxio** interpreter accepts the following commands.
|
|||
<tr>
|
||||
<td>copyFromLocal</td>
|
||||
<td>copyFromLocal "source path" "remote path"</td>
|
||||
<td>Copy the specified file specified by "source path" to the path specified by "remote path".
|
||||
<td>Copy the specified file specified by "source path" to the path specified by "remote path".
|
||||
This command will fail if "remote path" already exists.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -230,4 +230,4 @@ Following steps are performed:
|
|||
|
||||
<center>
|
||||
|
||||
</center>
|
||||
</center>
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ With the `search` command, you can send a search query to Elasticsearch. There a
|
|||
* You can provide a JSON-formatted query, that is exactly what you provide when you use the REST API of Elasticsearch.
|
||||
* See [Elasticsearch search API reference document](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) for more details about the content of the search queries.
|
||||
* You can also provide the content of a `query_string`.
|
||||
* This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }`
|
||||
* This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }`
|
||||
* See [Elasticsearch query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) for more details about the content of such a query.
|
||||
|
||||
```bash
|
||||
|
|
@ -119,10 +119,10 @@ Examples:
|
|||
```bash
|
||||
%elasticsearch
|
||||
search / { "query": { "match_all": { } } }
|
||||
|
||||
|
||||
%elasticsearch
|
||||
search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
|
||||
|
||||
|
||||
%elasticsearch
|
||||
search /logs { "aggs": {
|
||||
"content_length_stats": {
|
||||
|
|
@ -130,7 +130,7 @@ Examples:
|
|||
"field": "content_length"
|
||||
}
|
||||
}
|
||||
} }
|
||||
} }
|
||||
```
|
||||
|
||||
* With query_string elements:
|
||||
|
|
@ -138,7 +138,7 @@ Examples:
|
|||
```bash
|
||||
%elasticsearch
|
||||
search /logs request.method:GET AND status:200
|
||||
|
||||
|
||||
%elasticsearch
|
||||
search /logs (404 AND (POST OR DELETE))
|
||||
```
|
||||
|
|
@ -178,6 +178,9 @@ Examples:
|
|||
* With a JSON query:
|
||||
|
||||
|
||||
* With a JSON query containing a `fields` parameter (for filtering the fields in the response): in this case, all the fields values in the response are arrays, so, after flattening the result, the format of all the field names is `field_name[x]`
|
||||
|
||||
|
||||
* With a query string:
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ It supports the basic shell file commands applied to HDFS, it currently only sup
|
|||
|
||||
> **Tip :** Use ( Ctrl + . ) for autocompletion.
|
||||
|
||||
### Create Interpreter
|
||||
### Create Interpreter
|
||||
|
||||
In a notebook, to enable the **HDFS** interpreter, click the **Gear** icon and select **HDFS**.
|
||||
|
||||
|
|
@ -53,4 +53,4 @@ In a notebook, to enable the **HDFS** interpreter, click the **Gear** icon and s
|
|||
You can confirm that you're able to access the WebHDFS API by running a curl command against the WebHDFS end point provided to the interpreter.
|
||||
|
||||
Here is an example:
|
||||
$> curl "http://localhost:50070/webhdfs/v1/?op=LISTSTATUS"
|
||||
$> curl "http://localhost:50070/webhdfs/v1/?op=LISTSTATUS"
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilita
|
|||
<tr>
|
||||
<td>${prefix}.driver</td>
|
||||
<td></td>
|
||||
<td>Driver class path of <code>%hive(${prefix})</code> </td>
|
||||
<td>Driver class path of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.url</td>
|
||||
|
|
@ -93,9 +93,9 @@ You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html)
|
|||
|
||||
```sql
|
||||
%hive
|
||||
SELECT ${group_by}, count(*) as count
|
||||
FROM retail_demo.order_lineitems_pxf
|
||||
GROUP BY ${group_by=product_id,product_id|product_name|customer_id|store_id}
|
||||
ORDER BY count ${order=DESC,DESC|ASC}
|
||||
SELECT ${group_by}, count(*) as count
|
||||
FROM retail_demo.order_lineitems_pxf
|
||||
GROUP BY ${group_by=product_id,product_id|product_name|customer_id|store_id}
|
||||
ORDER BY count ${order=DESC,DESC|ASC}
|
||||
LIMIT ${limit=10};
|
||||
```
|
||||
|
|
|
|||
|
|
@ -18,17 +18,17 @@ You can use Zeppelin to retrieve distributed data from cache using Ignite SQL in
|
|||
### Installing and Running Ignite example
|
||||
In order to use Ignite interpreters, you may install Apache Ignite in some simple steps:
|
||||
|
||||
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zepplin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
|
||||
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
|
||||
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zeppelin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
|
||||
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
|
||||
|
||||
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
|
||||
* Set examples directory path to Eclipse and select the pom.xml.
|
||||
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
|
||||
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
|
||||
|
||||
> **Tip. If you want to run Ignite examples on the cli not IDE, you can export executable Jar file from IDE. Then run it by using below command.**
|
||||
|
||||
```
|
||||
$ nohup java -jar </path/to/your Jar file name>
|
||||
```
|
||||
$ nohup java -jar </path/to/your Jar file name>
|
||||
```
|
||||
|
||||
### Configuring Ignite Interpreter
|
||||
|
|
@ -78,17 +78,17 @@ For more interpreter binding information see [here](http://zeppelin.incubator.ap
|
|||
|
||||
### How to use Ignite SQL interpreter
|
||||
In order to execute SQL query, use ` %ignite.ignitesql ` prefix. <br>
|
||||
Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ).
|
||||
Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ).
|
||||
For example, you can select top 10 words in the words cache using the following query
|
||||
|
||||
```
|
||||
%ignite.ignitesql
|
||||
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
|
||||
```
|
||||
```
|
||||
%ignite.ignitesql
|
||||
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
|
||||
```
|
||||
|
||||
|
||||
|
||||
As long as your Ignite version and Zeppelin Ignite version is same, you can also use scala code. Please check the Zeppelin Ignite version before you download your own Ignite.
|
||||
As long as your Ignite version and Zeppelin Ignite version is same, you can also use scala code. Please check the Zeppelin Ignite version before you download your own Ignite.
|
||||
|
||||
```
|
||||
%ignite
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ group: manual
|
|||
In order to use Lens interpreters, you may install Apache Lens in some simple steps:
|
||||
|
||||
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
|
||||
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
|
||||
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
|
||||
3. Now, you can start lens server (or stop).
|
||||
|
||||
```
|
||||
|
|
@ -77,16 +77,16 @@ At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zep
|
|||
|
||||
|
||||
|
||||
### Interpreter Bindging for Zeppelin Notebook
|
||||
### Interpreter Binding for Zeppelin Notebook
|
||||
After configuring Lens interpreter, create your own notebook, then you can bind interpreters like below image.
|
||||
|
||||
|
||||
|
||||
|
||||
For more interpreter binding information see [here](http://zeppelin.incubator.apache.org/docs/manual/interpreters.html).
|
||||
|
||||
### How to use
|
||||
You can analyze your data by using [OLAP Cube](http://lens.apache.org/user/olap-cube.html) [QL](http://lens.apache.org/user/cli.html) which is a high level SQL like language to query and describe data sets organized in data cubes.
|
||||
You may experience OLAP Cube like this [Video tutorial](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
|
||||
### How to use
|
||||
You can analyze your data by using [OLAP Cube](http://lens.apache.org/user/olap-cube.html) [QL](http://lens.apache.org/user/cli.html) which is a high level SQL like language to query and describe data sets organized in data cubes.
|
||||
You may experience OLAP Cube like this [Video tutorial](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
|
||||
As you can see in this video, they are using Lens Client Shell(./bin/lens-cli.sh). All of these functions also can be used on Zeppelin by using Lens interpreter.
|
||||
|
||||
<li> Create and Use(Switch) Databases.
|
||||
|
|
@ -105,7 +105,7 @@ use newDb
|
|||
create storage your/path/to/lens/client/examples/resources/db-storage.xml
|
||||
```
|
||||
|
||||
<li> Create Dimensions, Show fields and join-chains of them.
|
||||
<li> Create Dimensions, Show fields and join-chains of them.
|
||||
|
||||
```
|
||||
create dimension your/path/to/lens/client/examples/resources/customer.xml
|
||||
|
|
@ -121,8 +121,8 @@ dimension show joinchains customer
|
|||
|
||||
<li> Create Caches, Show fields and join-chains of them.
|
||||
|
||||
```
|
||||
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
|
||||
```
|
||||
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
|
||||
```
|
||||
|
||||
```
|
||||
|
|
@ -133,7 +133,7 @@ cube show fields sales
|
|||
cube show joinchains sales
|
||||
```
|
||||
|
||||
<li> Create Dimtables and Fact.
|
||||
<li> Create Dimtables and Fact.
|
||||
|
||||
```
|
||||
create dimtable your/path/to/lens/client/examples/resources/customer_table.xml
|
||||
|
|
@ -163,7 +163,7 @@ query execute cube select customer_city_name, product_details.description, produ
|
|||
|
||||
These are just examples that provided in advance by Lens. If you want to explore whole tutorials of Lens, see the [tutorial video](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
|
||||
|
||||
### Lens UI Service
|
||||
### Lens UI Service
|
||||
Lens also provides web UI service. Once the server starts up, you can open the service on http://serverhost:19999/index.html and browse. You may also check the structure that you made and use query easily here.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,26 +15,26 @@ This is a the Apache (incubating) Zeppelin project, with the addition of support
|
|||
Additional requirements for the R interpreter are:
|
||||
|
||||
* R 3.1 or later (earlier versions may work, but have not been tested)
|
||||
* The `evaluate` R package.
|
||||
|
||||
* The `evaluate` R package.
|
||||
|
||||
For full R support, you will also need the following R packages:
|
||||
|
||||
* `knitr`
|
||||
|
||||
* `knitr`
|
||||
* `repr` -- available with `devtools::install_github("IRkernel/repr")`
|
||||
* `htmltools` -- required for some interactive plotting
|
||||
* `base64enc` -- required to view R base plots
|
||||
|
||||
### Configuration
|
||||
### Configuration
|
||||
|
||||
To run Zeppelin with the R Interpreter, the SPARK_HOME environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`.
|
||||
To run Zeppelin with the R Interpreter, the SPARK_HOME environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`.
|
||||
|
||||
If it is not set, the R Interpreter will not be able to interface with Spark.
|
||||
If it is not set, the R Interpreter will not be able to interface with Spark.
|
||||
|
||||
You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up.
|
||||
You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up.
|
||||
|
||||
### Using the R Interpreter
|
||||
|
||||
By default, the R Interpreter appears as two Zeppelin Interpreters, `%r` and `%knitr`.
|
||||
By default, the R Interpreter appears as two Zeppelin Interpreters, `%r` and `%knitr`.
|
||||
|
||||
`%r` will behave like an ordinary REPL. You can execute commands as in the CLI.
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ R base plotting is fully supported
|
|||
|
||||
[](screenshots/replhist.png)
|
||||
|
||||
If you return a data.frame, Zeppelin will attempt to display it using Zeppelin's built-in visualizations.
|
||||
If you return a data.frame, Zeppelin will attempt to display it using Zeppelin's built-in visualizations.
|
||||
|
||||
[](screenshots/replhead.png)
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ The two interpreters share the same environment. If you define a variable from
|
|||
If `SPARK_HOME` is set, the `SparkR` package will be loaded automatically:
|
||||
|
||||
[](screenshots/sparkrfaithful.png)
|
||||
|
||||
|
||||
The Spark Context and SQL Context are created and injected into the local environment automatically as `sc` and `sql`.
|
||||
|
||||
The same context are shared with the `%spark`, `%sql` and `%pyspark` interpreters:
|
||||
|
|
@ -79,9 +79,9 @@ And vice versa:
|
|||
|
||||
### Caveats & Troubleshooting
|
||||
|
||||
* Almost all issues with the R interpreter turned out to be caused by an incorrectly set `SPARK_HOME`. The R interpreter must load a version of the `SparkR` package that matches the running version of Spark, and it does this by searching `SPARK_HOME`. If Zeppelin isn't configured to interface with Spark in `SPARK_HOME`, the R interpreter will not be able to connect to Spark.
|
||||
* Almost all issues with the R interpreter turned out to be caused by an incorrectly set `SPARK_HOME`. The R interpreter must load a version of the `SparkR` package that matches the running version of Spark, and it does this by searching `SPARK_HOME`. If Zeppelin isn't configured to interface with Spark in `SPARK_HOME`, the R interpreter will not be able to connect to Spark.
|
||||
|
||||
* The `knitr` environment is persistent. If you run a chunk from Zeppelin that changes a variable, then run the same chunk again, the variable has already been changed. Use immutable variables.
|
||||
* The `knitr` environment is persistent. If you run a chunk from Zeppelin that changes a variable, then run the same chunk again, the variable has already been changed. Use immutable variables.
|
||||
|
||||
* (Note that `%spark.r` and `$r` are two different ways of calling the same interpreter, as are `%spark.knitr` and `%knitr`. By default, Zeppelin puts the R interpreters in the `%spark.` Interpreter Group.
|
||||
|
||||
|
|
@ -89,13 +89,13 @@ And vice versa:
|
|||
|
||||
* If you return a data.frame (for instance, from calling `head()`) from the `%spark.r` interpreter, it will be parsed by Zeppelin's built-in data visualization system.
|
||||
|
||||
* Why `knitr` Instead of `rmarkdown`? Why no `htmlwidgets`? In order to support `htmlwidgets`, which has indirect dependencies, `rmarkdown` uses `pandoc`, which requires writing to and reading from disc. This makes it many times slower than `knitr`, which can operate entirely in RAM.
|
||||
* Why `knitr` Instead of `rmarkdown`? Why no `htmlwidgets`? In order to support `htmlwidgets`, which has indirect dependencies, `rmarkdown` uses `pandoc`, which requires writing to and reading from disc. This makes it many times slower than `knitr`, which can operate entirely in RAM.
|
||||
|
||||
* Why no `ggvis` or `shiny`? Supporting `shiny` would require integrating a reverse-proxy into Zeppelin, which is a task.
|
||||
* Why no `ggvis` or `shiny`? Supporting `shiny` would require integrating a reverse-proxy into Zeppelin, which is a task.
|
||||
|
||||
* Max OS X & case-insensitive filesystem. If you try to install on a case-insensitive filesystem, which is the Mac OS X default, maven can unintentionally delete the install directory because `r` and `R` become the same subdirectory.
|
||||
* Max OS X & case-insensitive filesystem. If you try to install on a case-insensitive filesystem, which is the Mac OS X default, maven can unintentionally delete the install directory because `r` and `R` become the same subdirectory.
|
||||
|
||||
* Error `unable to start device X11` with the repl interpreter. Check your shell login scripts to see if they are adjusting the `DISPLAY` environment variable. This is common on some operating systems as a workaround for ssh issues, but can interfere with R plotting.
|
||||
* Error `unable to start device X11` with the repl interpreter. Check your shell login scripts to see if they are adjusting the `DISPLAY` environment variable. This is common on some operating systems as a workaround for ssh issues, but can interfere with R plotting.
|
||||
|
||||
* akka Library Version or `TTransport` errors. This can happen if you try to run Zeppelin with a SPARK_HOME that has a version of Spark other than the one specified with `-Pspark-1.x` when Zeppelin was compiled.
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ To run R code and visualize plots in Apache Zeppelin, you will need R on your ma
|
|||
|
||||
+ For Centos: `yum install R R-devel libcurl-devel openssl-devel`
|
||||
+ For Ubuntu: `apt-get install r-base`
|
||||
|
||||
|
||||
Validate your installation with a simple R command:
|
||||
|
||||
```
|
||||
|
|
@ -135,4 +135,3 @@ We recommend you to also install the following optional R libraries for happy da
|
|||
+ caret
|
||||
+ sqldf
|
||||
+ wordcloud
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,6 @@ If you click on the icon for the pie chart, you should be able to see a chart li
|
|||
|
||||
|
||||
### Current Status & Future Work
|
||||
The current implementation of the Scalding interpreter does not support canceling jobs, or fine-grained progress updates.
|
||||
The current implementation of the Scalding interpreter does not support canceling jobs, or fine-grained progress updates.
|
||||
|
||||
The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ group: manual
|
|||
|
||||
|
||||
## Spark Interpreter for Apache Zeppelin
|
||||
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
|
||||
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
|
||||
Spark Interpreter group, which consisted of 4 interpreters.
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -40,7 +40,7 @@ Spark Interpreter group, which consisted of 4 interpreters.
|
|||
</table>
|
||||
|
||||
## Configuration
|
||||
Zeppelin provides the below properties for Spark interpreter.
|
||||
Zeppelin provides the below properties for Spark interpreter.
|
||||
You can also set other Spark properties which are not listed in the table. If so, please refer to [Spark Available Properties](http://spark.apache.org/docs/latest/configuration.html#available-properties).
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
|
|
@ -273,13 +273,13 @@ z.put("objName", myObject)
|
|||
%pyspark
|
||||
myObject = z.get("objName")
|
||||
{% endhighlight %}
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
### Form Creation
|
||||
|
||||
ZeppelinContext provides functions for creating forms.
|
||||
ZeppelinContext provides functions for creating forms.
|
||||
In scala and python environments, you can create forms programmatically.
|
||||
<div class="codetabs">
|
||||
<div data-lang="scala" markdown="1">
|
||||
|
|
@ -306,13 +306,13 @@ z.select("formName", "option1", Seq(("option1", "option1DisplayName"),
|
|||
|
||||
{% highlight python %}
|
||||
%pyspark
|
||||
# Create text input form
|
||||
# Create text input form
|
||||
z.input("formName")
|
||||
|
||||
# Create text input form with default value
|
||||
# Create text input form with default value
|
||||
z.input("formName", "defaultValue")
|
||||
|
||||
# Create select form
|
||||
# Create select form
|
||||
z.select("formName", [("option1", "option1DisplayName"),
|
||||
("option2", "option2DisplayName")])
|
||||
|
||||
|
|
@ -320,7 +320,7 @@ z.select("formName", [("option1", "option1DisplayName"),
|
|||
z.select("formName", [("option1", "option1DisplayName"),
|
||||
("option2", "option2DisplayName")], "option1")
|
||||
{% endhighlight %}
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -334,9 +334,10 @@ select * from ${table=defaultTableName} where text like '%${search}%'
|
|||
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
|
||||
|
||||
|
||||
### Separate Interpreter for each note
|
||||
### Interpreter setting option.
|
||||
|
||||
Interpreter setting can choose one of 'shared', 'scoped', 'isolated' option. Spark interpreter creates separate scala compiler per each notebook but share a single SparkContext in 'scoped' mode (experimental). It creates separate SparkContext per each notebook in 'isolated' mode.
|
||||
|
||||
In 'Separate Interpreter for each note' mode, SparkInterpreter creates scala compiler per each notebook. However it still shares the single SparkContext.
|
||||
|
||||
## Setting up Zeppelin with Kerberos
|
||||
Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark on YARN:
|
||||
|
|
@ -359,5 +360,3 @@ This is to make the server communicate with KDC.
|
|||
> **NOTE:** If you do not have access to the above spark-defaults.conf file, optionally, you may add the lines to the Spark Interpreter through the Interpreter tab in the Zeppelin UI.
|
||||
|
||||
4. That's it. Play with Zeppelin !
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Dependnecy Management"
|
||||
title: "Dependency Management"
|
||||
description: ""
|
||||
group: manual
|
||||
---
|
||||
|
|
@ -71,4 +71,3 @@ When your code requires external library, instead of doing download/copy/restart
|
|||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -32,13 +32,13 @@ When you click the ```+Create``` button in the interpreter page, the interpreter
|
|||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
|
||||
|
||||
## What is Zeppelin Interpreter Setting?
|
||||
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
|
||||
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
|
||||
|
||||
Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
|
||||
|
||||
Each notebook can be binded to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
|
||||
Each notebook can be bound to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">
|
||||
|
||||
|
|
@ -56,8 +56,7 @@ Each interpreters is belonged to a single group and registered together. All of
|
|||
|
||||
## Interpreter binding mode
|
||||
|
||||
Each Interpreter Setting can choose one of two different interpreter binding mode.
|
||||
Shared mode (default) and 'Separate Interpreter for each note' mode. In shared mode, every notebook binded to the Interpreter Setting will share the single Interpreter instance. In 'Separate Interpreter for each note' mode, each notebook will create new Interpreter instance. Therefore each notebook will have fresh new Interpreter environment.
|
||||
Each Interpreter Setting can choose one of 'shared', 'scoped', 'isolated' interpreter binding mode.
|
||||
In 'shared' mode, every notebook bound to the Interpreter Setting will share the single Interpreter instance. In 'scoped' mode, each notebook will create new Interpreter instance in the same interpreter process. In 'isolated' mode, each notebook will create new Interpreter process.
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_persession.png" width="400px">
|
||||
|
||||
|
|
|
|||
|
|
@ -21,66 +21,66 @@ limitations under the License.
|
|||
|
||||
## Customize your zeppelin homepage
|
||||
Zeppelin allows you to use one of the notebooks you create as your zeppelin Homepage.
|
||||
With that you can brand your zeppelin installation,
|
||||
With that you can brand your zeppelin installation,
|
||||
adjust the instruction to your users needs and even translate to other languages.
|
||||
|
||||
<br />
|
||||
### How to set a notebook as your zeppelin homepage
|
||||
|
||||
The process for creating your homepage is very simple as shown below:
|
||||
|
||||
|
||||
1. Create a notebook using zeppelin
|
||||
2. Set the notebook id in the config file
|
||||
3. Restart zeppelin
|
||||
|
||||
|
||||
<br />
|
||||
#### Create a notebook using zeppelin
|
||||
Create a new notebook using zeppelin,
|
||||
you can use ```%md``` interpreter for markdown content or any other interpreter you like.
|
||||
|
||||
You can also use the display system to generate [text](../displaysystem/display.html),
|
||||
|
||||
You can also use the display system to generate [text](../displaysystem/display.html),
|
||||
[html](../displaysystem/display.html#html),[table](../displaysystem/table.html) or
|
||||
[angular](../displaysystem/angular.html)
|
||||
|
||||
Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide
|
||||
Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide
|
||||
the code sections.
|
||||
|
||||
|
||||
<br />
|
||||
#### Set the notebook id in the config file
|
||||
To set the notebook id in the config file you should copy it from the last word in the notebook url
|
||||
|
||||
To set the notebook id in the config file you should copy it from the last word in the notebook url
|
||||
|
||||
for example
|
||||
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_id.png" />
|
||||
|
||||
Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable
|
||||
or ```zeppelin.notebook.homescreen``` property.
|
||||
|
||||
You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable
|
||||
Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable
|
||||
or ```zeppelin.notebook.homescreen``` property.
|
||||
|
||||
You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable
|
||||
or ```zeppelin.notebook.homescreen.hide``` property to hide the new notebook from the notebook list.
|
||||
|
||||
<br />
|
||||
#### Restart zeppelin
|
||||
Restart your zeppelin server
|
||||
|
||||
|
||||
```
|
||||
./bin/zeppelin-deamon stop
|
||||
./bin/zeppelin-deamon stop
|
||||
./bin/zeppelin-deamon start
|
||||
```
|
||||
####That's it! Open your browser and navigate to zeppelin and see your customized homepage...
|
||||
|
||||
|
||||
|
||||
|
||||
<br />
|
||||
### Show notebooks list in your custom homepage
|
||||
If you want to display the list of notebooks on your custom zeppelin homepage all
|
||||
If you want to display the list of notebooks on your custom zeppelin homepage all
|
||||
you need to do is use our %angular support.
|
||||
|
||||
|
||||
<br />
|
||||
Add the following code to a paragraph in you home page and run it... walla! you have your notebooks list.
|
||||
|
||||
|
||||
```javascript
|
||||
println(
|
||||
"""%angular
|
||||
"""%angular
|
||||
<div class="col-md-4" ng-controller="HomeCtrl as home">
|
||||
<h4>Notebooks</h4>
|
||||
<div>
|
||||
|
|
@ -95,15 +95,15 @@ you need to do is use our %angular support.
|
|||
</div>
|
||||
""")
|
||||
```
|
||||
|
||||
|
||||
After running the notebook you will see output similar to this one:
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/homepage_notebook_list.png" />
|
||||
|
||||
|
||||
The main trick here relays in linking the ```<div>``` to the controller:
|
||||
|
||||
|
||||
```javascript
|
||||
<div class="col-md-4" ng-controller="HomeCtrl as home">
|
||||
```
|
||||
|
||||
|
||||
Once we have ```home``` as our controller variable in our ```<div></div>```
|
||||
we can use ```home.notes.list``` to get access to the notebook list.
|
||||
we can use ```home.notes.list``` to get access to the notebook list.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ limitations under the License.
|
|||
{% include JB/setup %}
|
||||
|
||||
# Shiro authentication for Apache Zeppelin
|
||||
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
|
||||
[Apache Shiro](http://shiro.apache.org/) is a powerful and easy-to-use Java security framework that performs authentication, authorization, cryptography, and session management. In this documentation, we will explain step by step how Shiro works for Zeppelin notebook authentication.
|
||||
|
||||
When you connect to Apache Zeppelin, you will be asked to enter your credentials. Once you logged in, then you have access to all notes including other user's notes.
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ When you connect to Apache Zeppelin, you will be asked to enter your credentials
|
|||
You can setup **Zeppelin notebook authentication** in some simple steps.
|
||||
|
||||
####1. Secure the HTTP channel
|
||||
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
|
||||
To secure the HTTP channel, you have to change both **anon** and **authcBasic** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authcBasic** means "basic auth security".
|
||||
|
||||
The default status of them is
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ The default status of them is
|
|||
/** = anon
|
||||
#/** = authcBasic
|
||||
```
|
||||
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
|
||||
Deactivate the line "/** = anon" and activate the line "/** = authcBasic" in `conf/shiro.ini` file.
|
||||
|
||||
```
|
||||
#/** = anon
|
||||
|
|
@ -49,15 +49,15 @@ For the further information about `shiro.ini` file format, please refer to [Shi
|
|||
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
|
||||
|
||||
####3. Start Zeppelin
|
||||
|
||||
|
||||
```
|
||||
bin/zeppelin-daemon.sh start (or restart)
|
||||
```
|
||||
|
||||
|
||||
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
|
||||
|
||||
####4. Login
|
||||
Finally, you can login using one of the below **username/password** combinations.
|
||||
Finally, you can login using one of the below **username/password** combinations.
|
||||
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png" width="40%" height="40%"></center>
|
||||
|
||||
|
|
@ -66,7 +66,7 @@ admin = password1
|
|||
user1 = password2
|
||||
user2 = password3
|
||||
```
|
||||
|
||||
|
||||
Those combinations are defined in the `conf/shiro.ini` file.
|
||||
|
||||
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/incubator-zeppelin/blob/master/SECURITY-README.md).
|
||||
> **NOTE :** This documentation is originally from [SECURITY-README.md](https://github.com/apache/incubator-zeppelin/blob/master/SECURITY-README.md).
|
||||
|
|
|
|||
|
|
@ -21,18 +21,18 @@ limitations under the License.
|
|||
|
||||
## Zeppelin REST API
|
||||
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
|
||||
|
||||
|
||||
All REST API are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
|
||||
|
||||
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
|
||||
|
||||
Note that Zeppelin REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
|
||||
[JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
|
||||
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
|
||||
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
|
||||
|
||||
<br />
|
||||
### Configuration REST API list
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -41,7 +41,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method return all key/value pair of configurations on the server.<br/>
|
||||
<td>This ```GET``` method return all key/value pair of configurations on the server.<br/>
|
||||
Note: For security reason, some pairs would not be shown.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -94,9 +94,9 @@ limitations under the License.
|
|||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -105,7 +105,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method return all prefix matched key/value pair of configurations on the server.<br/>
|
||||
<td>This ```GET``` method return all prefix matched key/value pair of configurations on the server.<br/>
|
||||
Note: For security reason, some pairs would not be shown.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -21,18 +21,18 @@ limitations under the License.
|
|||
|
||||
## Zeppelin REST API
|
||||
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
|
||||
|
||||
|
||||
All REST API are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
|
||||
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
|
||||
Note that Zeppelin REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
|
||||
[JSON View](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.incubator.apache.org/community.html).
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.incubator.apache.org/community.html).
|
||||
|
||||
<br />
|
||||
## Interpreter REST API List
|
||||
|
||||
|
||||
The role of registered interpreters, settings and interpreters group are described in [here](../manual/interpreters.html).
|
||||
|
||||
|
||||
### 1. List of Registered Interpreters & Interpreter Settings
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -106,9 +106,9 @@ limitations under the License.
|
|||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -268,8 +268,8 @@ limitations under the License.
|
|||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
### 3. Update an Interpreter Setting
|
||||
|
|
@ -354,7 +354,7 @@ limitations under the License.
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
<br/>
|
||||
### 4. Delete an Interpreter Setting
|
||||
|
||||
|
|
@ -388,9 +388,9 @@ limitations under the License.
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
<br/>
|
||||
### 5. Restart an Interpreter
|
||||
### 5. Restart an Interpreter
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
|
|
|
|||
|
|
@ -21,20 +21,20 @@ limitations under the License.
|
|||
|
||||
## Zeppelin REST API
|
||||
Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality.
|
||||
|
||||
|
||||
All REST APIs are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
|
||||
|
||||
|
||||
Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers
|
||||
such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
|
||||
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
|
||||
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
|
||||
|
||||
<br />
|
||||
### Notebook REST API list
|
||||
|
||||
Notebooks REST API supports the following operations: List, Create, Get, Delete, Clone, Run, Export, Import as detailed in the following table
|
||||
|
||||
|
||||
Notebooks REST API supports the following operations: List, Create, Get, Delete, Clone, Run, Export, Import as detailed in the following table
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -64,7 +64,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK","message":"","body":[{"name":"Homepage","id":"2AV4WUEMK"},{"name":"Zeppelin Tutorial","id":"2A94M5J1Z"}]}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -99,7 +99,7 @@ limitations under the License.
|
|||
<td> sample JSON input (with initial paragraphs) </td>
|
||||
<td><pre>
|
||||
{
|
||||
"name": "name of new notebook",
|
||||
"name": "name of new notebook",
|
||||
"paragraphs": [
|
||||
{
|
||||
"title": "paragraph title1",
|
||||
|
|
@ -118,7 +118,7 @@ limitations under the License.
|
|||
<td><pre>{"status": "CREATED","message": "","body": "2AZPHY918"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -220,7 +220,7 @@ limitations under the License.
|
|||
</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -251,9 +251,9 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK","message":""}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -262,7 +262,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method clones a notebook by the given id and create a new notebook using the given name
|
||||
<td>This ```POST``` method clones a notebook by the given id and create a new notebook using the given name
|
||||
or default name if none given.
|
||||
The body field of the returned JSON contains the new notebook id.
|
||||
</td>
|
||||
|
|
@ -288,7 +288,7 @@ limitations under the License.
|
|||
<td><pre>{"status": "CREATED","message": "","body": "2AZPHY918"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -319,7 +319,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -330,7 +330,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```DELETE``` method stops all paragraph in the given notebook id.
|
||||
<td>This ```DELETE``` method stops all paragraph in the given notebook id.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -350,9 +350,9 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -363,7 +363,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method gets all paragraph status by the given notebook id.
|
||||
<td>This ```GET``` method gets all paragraph status by the given notebook id.
|
||||
The body field of the returned JSON contains of the array that compose of the paragraph id, paragraph status, paragraph finish date, paragraph started date.
|
||||
</td>
|
||||
</tr>
|
||||
|
|
@ -384,7 +384,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK","body":[{"id":"20151121-212654_766735423","status":"FINISHED","finished":"Tue Nov 24 14:21:40 KST 2015","started":"Tue Nov 24 14:21:39 KST 2015"},{"progress":"1","id":"20151121-212657_730976687","status":"RUNNING","finished":"Tue Nov 24 14:21:35 KST 2015","started":"Tue Nov 24 14:21:40 KST 2015"}]}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -395,7 +395,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
|
||||
<td>This ```POST``` method runs the paragraph by given notebook and paragraph id.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -427,7 +427,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -438,7 +438,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```DELETE``` method stops the paragraph by given notebook and paragraph id.
|
||||
<td>This ```DELETE``` method stops the paragraph by given notebook and paragraph id.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -458,7 +458,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -469,7 +469,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method adds cron job by the given notebook id.
|
||||
<td>This ```POST``` method adds cron job by the given notebook id.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -493,7 +493,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -504,7 +504,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```DELETE``` method removes cron job by the given notebook id.
|
||||
<td>This ```DELETE``` method removes cron job by the given notebook id.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -524,7 +524,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -535,7 +535,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method gets cron job expression of given notebook id.
|
||||
<td>This ```GET``` method gets cron job expression of given notebook id.
|
||||
The body field of the returned JSON contains the cron expression.
|
||||
</td>
|
||||
</tr>
|
||||
|
|
@ -585,7 +585,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK", body: [{"id":"<noteId>/paragraph/<paragraphId>", "name":"Notebook Name", "snippet":"", "text":""}]}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
|
|
@ -616,16 +616,16 @@ limitations under the License.
|
|||
<tr>
|
||||
<td> sample JSON input (add to the last) </td>
|
||||
<td><pre>
|
||||
{
|
||||
"title": "Paragraph insert revised",
|
||||
"text": "%spark\nprintln(\"Paragraph insert revised\")"
|
||||
{
|
||||
"title": "Paragraph insert revised",
|
||||
"text": "%spark\nprintln(\"Paragraph insert revised\")"
|
||||
}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON input (add to specific index) </td>
|
||||
<td><pre>
|
||||
{
|
||||
"title": "Paragraph insert revised",
|
||||
{
|
||||
"title": "Paragraph insert revised",
|
||||
"text": "%spark\nprintln(\"Paragraph insert revised\")",
|
||||
"index": 0
|
||||
}
|
||||
|
|
@ -636,7 +636,7 @@ limitations under the License.
|
|||
<td><pre>{"status": "CREATED","message": "","body": "20151218-100330_1754029574"}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -709,7 +709,7 @@ limitations under the License.
|
|||
</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -740,7 +740,7 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK","message":""}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
|
|
@ -772,9 +772,9 @@ limitations under the License.
|
|||
<td><pre>{"status":"OK","message":""}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -826,7 +826,7 @@ limitations under the License.
|
|||
}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -881,4 +881,4 @@ limitations under the License.
|
|||
<td><pre>"status": "CREATED","message": "","body": "2AZPHY918"}</pre></td>
|
||||
</tr>
|
||||
</tr>
|
||||
</table>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -42,4 +42,4 @@ limitations under the License.
|
|||
</div>
|
||||
<div class="col-md-3">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ limitations under the License.
|
|||
-->
|
||||
# Authentication
|
||||
|
||||
Authentication is company-specific.
|
||||
Authentication is company-specific.
|
||||
|
||||
One option is to use [Basic Access Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication)
|
||||
|
||||
|
||||
### HTTP Basic Authentication using NGINX
|
||||
|
||||
> **Quote from Wikipedia:** NGINX is a web server. It can act as a reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer and an HTTP cache.
|
||||
|
|
@ -33,12 +33,12 @@ Here are instructions how to accomplish the setup NGINX as a front-end authentic
|
|||
This instruction based on Ubuntu 14.04 LTS but may work with other OS with few configuration changes.
|
||||
|
||||
1. Install NGINX server on your server instance
|
||||
|
||||
|
||||
You can install NGINX server with same machine where zeppelin installed or separate machine where it is dedicated to serve as proxy server.
|
||||
|
||||
```
|
||||
$ apt-get install nginx
|
||||
```
|
||||
```
|
||||
|
||||
1. Setup init script in NGINX
|
||||
|
||||
|
|
@ -53,11 +53,7 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
|
|||
|
||||
```
|
||||
upstream zeppelin {
|
||||
server [YOUR-ZEPPELIN-SERVER-IP]:8090;
|
||||
}
|
||||
|
||||
upstream zeppelin-wss {
|
||||
server [YOUR-ZEPPELIN-SERVER-IP]:8091;
|
||||
server [YOUR-ZEPPELIN-SERVER-IP]:8080;
|
||||
}
|
||||
|
||||
# Zeppelin Website
|
||||
|
|
@ -69,32 +65,23 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
|
|||
ssl_certificate [PATH-TO-YOUR-CERT-FILE]; # optional, to serve HTTPS connection
|
||||
ssl_certificate_key [PATH-TO-YOUR-CERT-KEY-FILE]; # optional, to serve HTTPS connection
|
||||
|
||||
if ($ssl_protocol = "") {
|
||||
if ($ssl_protocol = "") {
|
||||
rewrite ^ https://$host$request_uri? permanent; # optional, force to use HTTPS
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://zeppelin;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
proxy_pass http://zeppelin;
|
||||
proxy_redirect off;
|
||||
auth_basic "Restricted";
|
||||
auth_basic_user_file /etc/nginx/.htpasswd;
|
||||
}
|
||||
}
|
||||
|
||||
# Zeppelin Websocket
|
||||
server {
|
||||
listen [YOUR-ZEPPELIN-WEBSOCKET-PORT] ssl; # add ssl is optional, to serve HTTPS connection
|
||||
server_name [YOUR-ZEPPELIN-SERVER-HOST]; # for example: zeppelin.mycompany.com
|
||||
|
||||
ssl_certificate [PATH-TO-YOUR-CERT-FILE]; # optional, to serve HTTPS connection
|
||||
ssl_certificate_key [PATH-TO-YOUR-CERT-KEY-FILE]; # optional, to serve HTTPS connection
|
||||
|
||||
location / {
|
||||
proxy_pass http://zeppelin-wss;
|
||||
location /ws {
|
||||
proxy_pass http://zeppelin;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade websocket;
|
||||
proxy_set_header Connection upgrade;
|
||||
|
|
@ -104,7 +91,7 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
|
|||
```
|
||||
|
||||
Then make a symbolic link to this file from `/etc/nginx/sites-enabled/` to enable configuration above when NGINX reloads.
|
||||
|
||||
|
||||
```
|
||||
$ ln -s /etc/nginx/sites-enabled/my-basic-auth /etc/nginx/sites-available/my-basic-auth
|
||||
```
|
||||
|
|
@ -141,7 +128,5 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
|
|||
|
||||
Another option is to have an authentication server that can verify user credentials in an LDAP server.
|
||||
If an incoming request to the Zeppelin server does not have a cookie with user information encrypted with the authentication server public key, the user
|
||||
is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific
|
||||
URL in the Zeppelin server which sets the authentication cookie in the browser.
|
||||
The end result is that all requests to the Zeppelin
|
||||
web server have the authentication cookie which contains user and groups information.
|
||||
is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific URL in the Zeppelin server which sets the authentication cookie in the browser.
|
||||
The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.
|
||||
|
|
|
|||
|
|
@ -33,5 +33,5 @@ Before executing a Note operation, it checks if the user and the groups associat
|
|||
operation, it checks if the user and the groups have at least one entity that belongs to the reader entities.
|
||||
|
||||
To initialize and modify note permissions, we provide UI like "Interpreter binding". The user inputs comma separated entities for owners, readers and writers.
|
||||
We execute a rest api call with this information. In the backend we get the user information for the connection and allow the operation if the user and groups
|
||||
We execute a rest api call with this information. In the backend we get the user information for the connection and allow the operation if the user and groups
|
||||
associated with the current user have at least one entity that belongs to owner entities for the note.
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ title : Sitemap
|
|||
{% for page in site.pages %}
|
||||
{{site.production_url}}{{ page.url }}{% endfor %}
|
||||
{% for post in site.posts %}
|
||||
{{site.production_url}}{{ post.url }}{% endfor %}
|
||||
{{site.production_url}}{{ post.url }}{% endfor %}
|
||||
|
|
|
|||
|
|
@ -20,12 +20,12 @@ limitations under the License.
|
|||
### Notebook Storage
|
||||
|
||||
Zeppelin has a pluggable notebook storage mechanism controlled by `zeppelin.notebook.storage` configuration option with multiple implementations.
|
||||
There are few Notebook storages avaialble for a use out of the box:
|
||||
There are few Notebook storages available for a use out of the box:
|
||||
- (default) all notes are saved in the notebook folder in your local File System - `VFSNotebookRepo`
|
||||
- there is also an option to version it using local Git repository - `GitNotebookRepo`
|
||||
- another option is Amazon S3 service - `S3NotebookRepo`
|
||||
|
||||
Multiple storages can be used at the same time by providing a comma-separated list of the calss-names in the confiruration.
|
||||
Multiple storages can be used at the same time by providing a comma-separated list of the class-names in the configuration.
|
||||
By default, only first two of them will be automatically kept in sync by Zeppelin.
|
||||
|
||||
</br>
|
||||
|
|
@ -44,7 +44,7 @@ To enable versioning for all your local notebooks though a standard Git reposito
|
|||
</br>
|
||||
#### Notebook Storage in S3 <a name="S3"></a>
|
||||
|
||||
For notebook storage in S3 you need the AWS credentials, for this there are three options, the enviroment variable ```AWS_ACCESS_KEY_ID``` and ```AWS_ACCESS_SECRET_KEY```, credentials file in the folder .aws in you home and IAM role for your instance. For complete the need steps is necessary:
|
||||
For notebook storage in S3 you need the AWS credentials, for this there are three options, the environment variable ```AWS_ACCESS_KEY_ID``` and ```AWS_ACCESS_SECRET_KEY```, credentials file in the folder .aws in you home and IAM role for your instance. For complete the need steps is necessary:
|
||||
|
||||
</br>
|
||||
you need the following folder structure on S3
|
||||
|
|
@ -56,14 +56,14 @@ bucket_name/
|
|||
|
||||
```
|
||||
|
||||
set the enviroment variable in the file **zeppelin-env.sh**:
|
||||
set the environment variable in the file **zeppelin-env.sh**:
|
||||
|
||||
```
|
||||
export ZEPPELIN_NOTEBOOK_S3_BUCKET = bucket_name
|
||||
export ZEPPELIN_NOTEBOOK_S3_USER = username
|
||||
```
|
||||
|
||||
in the file **zeppelin-site.xml** uncommet and complete the next property:
|
||||
in the file **zeppelin-site.xml** uncomment and complete the next property:
|
||||
|
||||
```
|
||||
<!--If used S3 to storage, it is necessary the following folder structure bucket_name/username/notebook/-->
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@
|
|||
<properties>
|
||||
<elasticsearch.version>2.1.0</elasticsearch.version>
|
||||
<guava.version>18.0</guava.version>
|
||||
<json-flattener.version>0.1.1</json-flattener.version>
|
||||
<json-flattener.version>0.1.6</json-flattener.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
|
|||
|
|
@ -17,10 +17,21 @@
|
|||
|
||||
package org.apache.zeppelin.elasticsearch;
|
||||
|
||||
import com.github.wnameless.json.flattener.JsonFlattener;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonParseException;
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
|
|
@ -39,6 +50,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
|||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.search.SearchHitField;
|
||||
import org.elasticsearch.search.aggregations.Aggregation;
|
||||
import org.elasticsearch.search.aggregations.Aggregations;
|
||||
import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
|
||||
|
|
@ -48,9 +60,10 @@ import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.util.*;
|
||||
import com.github.wnameless.json.flattener.JsonFlattener;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonParseException;
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -80,7 +93,9 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
|
||||
private static final List<String> COMMANDS = Arrays.asList(
|
||||
"count", "delete", "get", "help", "index", "search");
|
||||
|
||||
|
||||
private static final Pattern FIELD_NAME_PATTERN = Pattern.compile("\\[\\\\\"(.+)\\\\\"\\](.*)");
|
||||
|
||||
|
||||
public static final String ELASTICSEARCH_HOST = "elasticsearch.host";
|
||||
public static final String ELASTICSEARCH_PORT = "elasticsearch.port";
|
||||
|
|
@ -141,7 +156,7 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
@Override
|
||||
public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) {
|
||||
logger.info("Run Elasticsearch command '" + cmd + "'");
|
||||
|
||||
|
||||
if (StringUtils.isEmpty(cmd) || StringUtils.isEmpty(cmd.trim())) {
|
||||
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
|
||||
}
|
||||
|
|
@ -260,15 +275,15 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
|
||||
/**
|
||||
* Processes a "get" request.
|
||||
*
|
||||
*
|
||||
* @param urlItems Items of the URL
|
||||
* @return Result of the get request, it contains a JSON-formatted string
|
||||
*/
|
||||
private InterpreterResult processGet(String[] urlItems) {
|
||||
|
||||
if (urlItems.length != 3
|
||||
|| StringUtils.isEmpty(urlItems[0])
|
||||
|| StringUtils.isEmpty(urlItems[1])
|
||||
if (urlItems.length != 3
|
||||
|| StringUtils.isEmpty(urlItems[0])
|
||||
|| StringUtils.isEmpty(urlItems[1])
|
||||
|| StringUtils.isEmpty(urlItems[2])) {
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR,
|
||||
"Bad URL (it should be /index/type/id)");
|
||||
|
|
@ -285,13 +300,13 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
InterpreterResult.Type.TEXT,
|
||||
json);
|
||||
}
|
||||
|
||||
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR, "Document not found");
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes a "count" request.
|
||||
*
|
||||
*
|
||||
* @param urlItems Items of the URL
|
||||
* @param data May contains the JSON of the request
|
||||
* @return Result of the count request, it contains the total hits
|
||||
|
|
@ -313,7 +328,7 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
|
||||
/**
|
||||
* Processes a "search" request.
|
||||
*
|
||||
*
|
||||
* @param urlItems Items of the URL
|
||||
* @param data May contains the JSON of the request
|
||||
* @param size Limit of result set
|
||||
|
|
@ -325,7 +340,7 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
return new InterpreterResult(InterpreterResult.Code.ERROR,
|
||||
"Bad URL (it should be /index1,index2,.../type1,type2,...)");
|
||||
}
|
||||
|
||||
|
||||
final SearchResponse response = searchData(urlItems, data, size);
|
||||
|
||||
return buildResponseMessage(response);
|
||||
|
|
@ -333,18 +348,18 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
|
||||
/**
|
||||
* Processes a "index" request.
|
||||
*
|
||||
*
|
||||
* @param urlItems Items of the URL
|
||||
* @param data JSON to be indexed
|
||||
* @return Result of the index request, it contains the id of the document
|
||||
*/
|
||||
private InterpreterResult processIndex(String[] urlItems, String data) {
|
||||
|
||||
|
||||
if (urlItems.length < 2 || urlItems.length > 3) {
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR,
|
||||
"Bad URL (it should be /index/type or /index/type/id)");
|
||||
}
|
||||
|
||||
|
||||
final IndexResponse response = client
|
||||
.prepareIndex(urlItems[0], urlItems[1], urlItems.length == 2 ? null : urlItems[2])
|
||||
.setSource(data)
|
||||
|
|
@ -358,15 +373,15 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
|
||||
/**
|
||||
* Processes a "delete" request.
|
||||
*
|
||||
*
|
||||
* @param urlItems Items of the URL
|
||||
* @return Result of the delete request, it contains the id of the deleted document
|
||||
*/
|
||||
private InterpreterResult processDelete(String[] urlItems) {
|
||||
|
||||
if (urlItems.length != 3
|
||||
|| StringUtils.isEmpty(urlItems[0])
|
||||
|| StringUtils.isEmpty(urlItems[1])
|
||||
if (urlItems.length != 3
|
||||
|| StringUtils.isEmpty(urlItems[0])
|
||||
|| StringUtils.isEmpty(urlItems[1])
|
||||
|| StringUtils.isEmpty(urlItems[2])) {
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR,
|
||||
"Bad URL (it should be /index/type/id)");
|
||||
|
|
@ -375,23 +390,23 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
final DeleteResponse response = client
|
||||
.prepareDelete(urlItems[0], urlItems[1], urlItems[2])
|
||||
.get();
|
||||
|
||||
|
||||
if (response.isFound()) {
|
||||
return new InterpreterResult(
|
||||
InterpreterResult.Code.SUCCESS,
|
||||
InterpreterResult.Type.TEXT,
|
||||
response.getId());
|
||||
}
|
||||
|
||||
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR, "Document not found");
|
||||
}
|
||||
|
||||
|
||||
private SearchResponse searchData(String[] urlItems, String query, int size) {
|
||||
|
||||
final SearchRequestBuilder reqBuilder = new SearchRequestBuilder(
|
||||
client, SearchAction.INSTANCE);
|
||||
reqBuilder.setIndices();
|
||||
|
||||
|
||||
if (urlItems.length >= 1) {
|
||||
reqBuilder.setIndices(StringUtils.split(urlItems[0], ","));
|
||||
}
|
||||
|
|
@ -452,18 +467,42 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
private String buildSearchHitsResponseMessage(SearchHit[] hits) {
|
||||
|
||||
|
||||
if (hits == null || hits.length == 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
//First : get all the keys in order to build an ordered list of the values for each hit
|
||||
//
|
||||
final Map<String, Object> hitFields = new HashMap<>();
|
||||
final List<Map<String, Object>> flattenHits = new LinkedList<>();
|
||||
final Set<String> keys = new TreeSet<>();
|
||||
for (SearchHit hit : hits) {
|
||||
final String json = hit.getSourceAsString();
|
||||
final Map<String, Object> flattenMap = JsonFlattener.flattenAsMap(json);
|
||||
// Fields can be found either in _source, or in fields (it depends on the query)
|
||||
//
|
||||
String json = hit.getSourceAsString();
|
||||
if (json == null) {
|
||||
hitFields.clear();
|
||||
for (SearchHitField hitField : hit.getFields().values()) {
|
||||
hitFields.put(hitField.getName(), hitField.getValues());
|
||||
}
|
||||
json = gson.toJson(hitFields);
|
||||
}
|
||||
|
||||
final Map<String, Object> flattenJsonMap = JsonFlattener.flattenAsMap(json);
|
||||
final Map<String, Object> flattenMap = new HashMap<>();
|
||||
for (Iterator<String> iter = flattenJsonMap.keySet().iterator(); iter.hasNext(); ) {
|
||||
// Replace keys that match a format like that : [\"keyname\"][0]
|
||||
final String fieldName = iter.next();
|
||||
final Matcher fieldNameMatcher = FIELD_NAME_PATTERN.matcher(fieldName);
|
||||
if (fieldNameMatcher.matches()) {
|
||||
flattenMap.put(fieldNameMatcher.group(1) + fieldNameMatcher.group(2),
|
||||
flattenJsonMap.get(fieldName));
|
||||
}
|
||||
else {
|
||||
flattenMap.put(fieldName, flattenJsonMap.get(fieldName));
|
||||
}
|
||||
}
|
||||
flattenHits.add(flattenMap);
|
||||
|
||||
for (String key : flattenMap.keySet()) {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,15 @@
|
|||
|
||||
package org.apache.zeppelin.elasticsearch;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
|
|
@ -29,21 +38,12 @@ import org.junit.AfterClass;
|
|||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class ElasticsearchInterpreterTest {
|
||||
|
||||
|
||||
private static Client elsClient;
|
||||
private static Node elsNode;
|
||||
private static ElasticsearchInterpreter interpreter;
|
||||
|
||||
|
||||
private static final String[] METHODS = { "GET", "PUT", "DELETE", "POST" };
|
||||
private static final int[] STATUS = { 200, 404, 500, 403 };
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ public class ElasticsearchInterpreterTest {
|
|||
.field("type", "integer")
|
||||
.endObject()
|
||||
.endObject().endObject().endObject()).get();
|
||||
|
||||
|
||||
for (int i = 0; i < 50; i++) {
|
||||
elsClient.prepareIndex("logs", "http", "" + i)
|
||||
.setRefresh(true)
|
||||
|
|
@ -100,7 +100,7 @@ public class ElasticsearchInterpreterTest {
|
|||
interpreter = new ElasticsearchInterpreter(props);
|
||||
interpreter.open();
|
||||
}
|
||||
|
||||
|
||||
@AfterClass
|
||||
public static void clean() {
|
||||
if (interpreter != null) {
|
||||
|
|
@ -116,41 +116,44 @@ public class ElasticsearchInterpreterTest {
|
|||
elsNode.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testCount() {
|
||||
|
||||
|
||||
InterpreterResult res = interpreter.interpret("count /unknown", null);
|
||||
assertEquals(Code.ERROR, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("count /logs", null);
|
||||
assertEquals("50", res.message());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGet() {
|
||||
|
||||
|
||||
InterpreterResult res = interpreter.interpret("get /logs/http/unknown", null);
|
||||
assertEquals(Code.ERROR, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("get /logs/http/10", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSearch() {
|
||||
|
||||
|
||||
InterpreterResult res = interpreter.interpret("size 10\nsearch /logs *", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("search /logs {{{hello}}}", null);
|
||||
assertEquals(Code.ERROR, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("search /logs { \"query\": { \"match\": { \"status\": 500 } } }", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
|
||||
res = interpreter.interpret("search /logs status:404", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
|
||||
res = interpreter.interpret("search /logs { \"fields\": [ \"date\", \"request.headers\" ], \"query\": { \"match\": { \"status\": 500 } } }", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -177,23 +180,23 @@ public class ElasticsearchInterpreterTest {
|
|||
" { \"terms\" : { \"field\" : \"status\" } } } }", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testIndex() {
|
||||
|
||||
|
||||
InterpreterResult res = interpreter.interpret("index /logs { \"date\": \"" + new Date() + "\", \"method\": \"PUT\", \"status\": \"500\" }", null);
|
||||
assertEquals(Code.ERROR, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("index /logs/http { \"date\": \"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }", null);
|
||||
assertEquals(Code.SUCCESS, res.code());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDelete() {
|
||||
|
||||
|
||||
InterpreterResult res = interpreter.interpret("delete /logs/http/unknown", null);
|
||||
assertEquals(Code.ERROR, res.code());
|
||||
|
||||
|
||||
res = interpreter.interpret("delete /logs/http/11", null);
|
||||
assertEquals("11", res.message());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -262,7 +262,7 @@ public class DepInterpreter extends Interpreter {
|
|||
public List<String> completion(String buf, int cursor) {
|
||||
ScalaCompleter c = completor.completer();
|
||||
Candidates ret = c.complete(buf, cursor);
|
||||
return scala.collection.JavaConversions.asJavaList(ret.candidates());
|
||||
return scala.collection.JavaConversions.seqAsJavaList(ret.candidates());
|
||||
}
|
||||
|
||||
private List<File> currentClassPath() {
|
||||
|
|
|
|||
|
|
@ -676,7 +676,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
}
|
||||
ScalaCompleter c = completor.completer();
|
||||
Candidates ret = c.complete(completionText, cursor);
|
||||
return scala.collection.JavaConversions.asJavaList(ret.candidates());
|
||||
return scala.collection.JavaConversions.seqAsJavaList(ret.candidates());
|
||||
}
|
||||
|
||||
private String getCompletionTargetString(String text, int cursor) {
|
||||
|
|
@ -924,14 +924,14 @@ public class SparkInterpreter extends Interpreter {
|
|||
|
||||
Object completedTaskInfo = null;
|
||||
|
||||
completedTaskInfo = JavaConversions.asJavaMap(
|
||||
completedTaskInfo = JavaConversions.mapAsJavaMap(
|
||||
(HashMap<Object, Object>) sparkListener.getClass()
|
||||
.getMethod("stageIdToTasksComplete").invoke(sparkListener)).get(id);
|
||||
|
||||
if (completedTaskInfo != null) {
|
||||
completedTasks += (int) completedTaskInfo;
|
||||
}
|
||||
List<Object> parents = JavaConversions.asJavaList((Seq<Object>) stage.getClass()
|
||||
List<Object> parents = JavaConversions.seqAsJavaList((Seq<Object>) stage.getClass()
|
||||
.getMethod("parents").invoke(stage));
|
||||
if (parents != null) {
|
||||
for (Object s : parents) {
|
||||
|
|
@ -960,7 +960,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
|
||||
Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks");
|
||||
Set<Tuple2<Object, Object>> keys =
|
||||
JavaConverters.asJavaSetConverter(stageIdData.keySet()).asJava();
|
||||
JavaConverters.setAsJavaSetConverter(stageIdData.keySet()).asJava();
|
||||
for (Tuple2<Object, Object> k : keys) {
|
||||
if (id == (int) k._1()) {
|
||||
Object uiData = stageIdData.get(k).get();
|
||||
|
|
@ -971,7 +971,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
logger.error("Error on getting progress information", e);
|
||||
}
|
||||
|
||||
List<Object> parents = JavaConversions.asJavaList((Seq<Object>) stage.getClass()
|
||||
List<Object> parents = JavaConversions.seqAsJavaList((Seq<Object>) stage.getClass()
|
||||
.getMethod("parents").invoke(stage));
|
||||
if (parents != null) {
|
||||
for (Object s : parents) {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ package org.apache.zeppelin.spark;
|
|||
|
||||
import static scala.collection.JavaConversions.asJavaCollection;
|
||||
import static scala.collection.JavaConversions.asJavaIterable;
|
||||
import static scala.collection.JavaConversions.asScalaIterable;
|
||||
import static scala.collection.JavaConversions.collectionAsScalaIterable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
|
|
@ -95,13 +95,13 @@ public class ZeppelinContext {
|
|||
for (Tuple2<Object, String> option : asJavaIterable(options)) {
|
||||
allChecked.add(option._1());
|
||||
}
|
||||
return checkbox(name, asScalaIterable(allChecked), options);
|
||||
return checkbox(name, collectionAsScalaIterable(allChecked), options);
|
||||
}
|
||||
|
||||
public scala.collection.Iterable<Object> checkbox(String name,
|
||||
scala.collection.Iterable<Object> defaultChecked,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return asScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
return collectionAsScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
tuplesToParamOptions(options)));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ The following components are provided under Apache License.
|
|||
(Apache 2.0) Jackson-dataformat-CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.2 - http://wiki.fasterxml.com/JacksonForCbor)
|
||||
(Apache 2.0) Jackson-dataformat-Smile (com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.6.2 - http://wiki.fasterxml.com/JacksonForSmile)
|
||||
(Apache 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.6.2 - https://github.com/FasterXML/jackson)
|
||||
(Apache 2.0) json-flattener (com.github.wnameless:json-flattener:0.1.1 - https://github.com/wnameless/json-flattener)
|
||||
(Apache 2.0) json-flattener (com.github.wnameless:json-flattener:0.1.6 - https://github.com/wnameless/json-flattener)
|
||||
(Apache 2.0) Spatial4J (com.spatial4j:spatial4j:0.4.1 - https://github.com/spatial4j/spatial4j)
|
||||
(Apache 2.0) T-Digest (com.tdunning:t-digest:3.0 - https://github.com/tdunning/t-digest)
|
||||
(Apache 2.0) Netty (io.netty:netty:3.10.5.Final - http://netty.io/)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
|
||||
<properties>
|
||||
<cxf.version>2.7.7</cxf.version>
|
||||
<jetty.version>8.1.14.v20131031</jetty.version>
|
||||
<jetty.version>9.2.15.v20160210</jetty.version>
|
||||
<commons.httpclient.version>4.3.6</commons.httpclient.version>
|
||||
</properties>
|
||||
|
||||
|
|
@ -129,11 +129,16 @@
|
|||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty.aggregate</groupId>
|
||||
<artifactId>jetty-all-server</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty.websocket</groupId>
|
||||
<artifactId>websocket-server</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
</dependency>
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-jsp</artifactId>
|
||||
|
|
@ -165,7 +170,7 @@
|
|||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
|
|
@ -237,6 +242,10 @@
|
|||
<groupId>xml-apis</groupId>
|
||||
<artifactId>xml-apis</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jetty.websocket</groupId>
|
||||
<artifactId>websocket-client</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -94,14 +94,13 @@ public class InterpreterRestApi {
|
|||
NewInterpreterSettingRequest.class);
|
||||
Properties p = new Properties();
|
||||
p.putAll(request.getProperties());
|
||||
InterpreterGroup interpreterGroup = interpreterFactory.add(request.getName(),
|
||||
InterpreterSetting interpreterSetting = interpreterFactory.add(request.getName(),
|
||||
request.getGroup(),
|
||||
request.getDependencies(),
|
||||
request.getOption(),
|
||||
p);
|
||||
InterpreterSetting setting = interpreterFactory.get(interpreterGroup.getId());
|
||||
logger.info("new setting created with {}", setting.id());
|
||||
return new JsonResponse(Status.CREATED, "", setting).build();
|
||||
logger.info("new setting created with {}", interpreterSetting.id());
|
||||
return new JsonResponse(Status.CREATED, "", interpreterSetting).build();
|
||||
} catch (InterpreterException e) {
|
||||
logger.error("Exception in InterpreterRestApi while creating ", e);
|
||||
return new JsonResponse(
|
||||
|
|
|
|||
|
|
@ -44,13 +44,10 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
|
|||
import org.apache.zeppelin.search.LuceneSearch;
|
||||
import org.apache.zeppelin.search.SearchService;
|
||||
import org.apache.zeppelin.socket.NotebookServer;
|
||||
import org.eclipse.jetty.server.AbstractConnector;
|
||||
import org.eclipse.jetty.server.Handler;
|
||||
import org.eclipse.jetty.server.Server;
|
||||
import org.eclipse.jetty.http.HttpVersion;
|
||||
import org.eclipse.jetty.server.*;
|
||||
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
|
||||
import org.eclipse.jetty.server.nio.SelectChannelConnector;
|
||||
import org.eclipse.jetty.server.session.SessionHandler;
|
||||
import org.eclipse.jetty.server.ssl.SslSelectChannelConnector;
|
||||
import org.eclipse.jetty.servlet.DefaultServlet;
|
||||
import org.eclipse.jetty.servlet.FilterHolder;
|
||||
import org.eclipse.jetty.servlet.ServletContextHandler;
|
||||
|
|
@ -62,7 +59,6 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
/**
|
||||
* Main class of Zeppelin.
|
||||
*
|
||||
*/
|
||||
public class ZeppelinServer extends Application {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinServer.class);
|
||||
|
|
@ -107,24 +103,26 @@ public class ZeppelinServer extends Application {
|
|||
}
|
||||
|
||||
public static void main(String[] args) throws InterruptedException {
|
||||
|
||||
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
|
||||
conf.setProperty("args", args);
|
||||
|
||||
// REST api
|
||||
final ServletContextHandler restApiContext = setupRestApiContextHandler(conf);
|
||||
jettyWebServer = setupJettyServer(conf);
|
||||
|
||||
// Notebook server
|
||||
final ServletContextHandler notebookContext = setupNotebookServer(conf);
|
||||
ContextHandlerCollection contexts = new ContextHandlerCollection();
|
||||
jettyWebServer.setHandler(contexts);
|
||||
|
||||
// Web UI
|
||||
final WebAppContext webApp = setupWebAppContext(conf);
|
||||
final WebAppContext webApp = setupWebAppContext(contexts, conf);
|
||||
|
||||
// add all handlers
|
||||
ContextHandlerCollection contexts = new ContextHandlerCollection();
|
||||
contexts.setHandlers(new Handler[]{restApiContext, notebookContext, webApp});
|
||||
// REST api
|
||||
setupRestApiContextHandler(webApp, conf);
|
||||
|
||||
jettyWebServer = setupJettyServer(conf);
|
||||
jettyWebServer.setHandler(contexts);
|
||||
// Notebook server
|
||||
setupNotebookServer(webApp, conf);
|
||||
|
||||
//Below is commented since zeppelin-docs module is removed.
|
||||
//final WebAppContext webAppSwagg = setupWebAppSwagger(conf);
|
||||
|
||||
LOG.info("Starting zeppelin server");
|
||||
try {
|
||||
|
|
@ -166,27 +164,50 @@ public class ZeppelinServer extends Application {
|
|||
}
|
||||
|
||||
private static Server setupJettyServer(ZeppelinConfiguration conf) {
|
||||
AbstractConnector connector;
|
||||
|
||||
final Server server = new Server();
|
||||
ServerConnector connector;
|
||||
|
||||
if (conf.useSsl()) {
|
||||
connector = new SslSelectChannelConnector(getSslContextFactory(conf));
|
||||
|
||||
HttpConfiguration httpConfig = new HttpConfiguration();
|
||||
httpConfig.setSecureScheme("https");
|
||||
httpConfig.setSecurePort(conf.getServerPort());
|
||||
httpConfig.setOutputBufferSize(32768);
|
||||
|
||||
HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
|
||||
SecureRequestCustomizer src = new SecureRequestCustomizer();
|
||||
// Only with Jetty 9.3.x
|
||||
// src.setStsMaxAge(2000);
|
||||
// src.setStsIncludeSubDomains(true);
|
||||
httpsConfig.addCustomizer(src);
|
||||
|
||||
connector = new ServerConnector(
|
||||
server,
|
||||
new SslConnectionFactory(getSslContextFactory(conf), HttpVersion.HTTP_1_1.asString()),
|
||||
new HttpConnectionFactory(httpsConfig));
|
||||
|
||||
|
||||
} else {
|
||||
connector = new SelectChannelConnector();
|
||||
|
||||
connector = new ServerConnector(server);
|
||||
|
||||
}
|
||||
|
||||
// Set some timeout options to make debugging easier.
|
||||
int timeout = 1000 * 30;
|
||||
connector.setMaxIdleTime(timeout);
|
||||
connector.setIdleTimeout(timeout);
|
||||
connector.setSoLingerTime(-1);
|
||||
connector.setHost(conf.getServerAddress());
|
||||
connector.setPort(conf.getServerPort());
|
||||
|
||||
final Server server = new Server();
|
||||
server.addConnector(connector);
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
private static ServletContextHandler setupNotebookServer(ZeppelinConfiguration conf) {
|
||||
private static void setupNotebookServer(WebAppContext webapp,
|
||||
ZeppelinConfiguration conf) {
|
||||
notebookWsServer = new NotebookServer();
|
||||
String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
|
||||
final ServletHolder servletHolder = new ServletHolder(notebookWsServer);
|
||||
|
|
@ -195,28 +216,23 @@ public class ZeppelinServer extends Application {
|
|||
final ServletContextHandler cxfContext = new ServletContextHandler(
|
||||
ServletContextHandler.SESSIONS);
|
||||
|
||||
cxfContext.setSessionHandler(new SessionHandler());
|
||||
cxfContext.setContextPath(conf.getServerContextPath());
|
||||
cxfContext.addServlet(servletHolder, "/ws/*");
|
||||
cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*",
|
||||
EnumSet.allOf(DispatcherType.class));
|
||||
return cxfContext;
|
||||
webapp.addServlet(servletHolder, "/ws/*");
|
||||
webapp.addFilter(new FilterHolder(CorsFilter.class), "/*",
|
||||
EnumSet.allOf(DispatcherType.class));
|
||||
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private static SslContextFactory getSslContextFactory(ZeppelinConfiguration conf) {
|
||||
// Note that the API for the SslContextFactory is different for
|
||||
// Jetty version 9
|
||||
SslContextFactory sslContextFactory = new SslContextFactory();
|
||||
|
||||
// Set keystore
|
||||
sslContextFactory.setKeyStore(conf.getKeyStorePath());
|
||||
sslContextFactory.setKeyStorePath(conf.getKeyStorePath());
|
||||
sslContextFactory.setKeyStoreType(conf.getKeyStoreType());
|
||||
sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword());
|
||||
sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword());
|
||||
|
||||
// Set truststore
|
||||
sslContextFactory.setTrustStore(conf.getTrustStorePath());
|
||||
sslContextFactory.setTrustStorePath(conf.getTrustStorePath());
|
||||
sslContextFactory.setTrustStoreType(conf.getTrustStoreType());
|
||||
sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword());
|
||||
|
||||
|
|
@ -225,43 +241,31 @@ public class ZeppelinServer extends Application {
|
|||
return sslContextFactory;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused") //TODO(bzz) why unused?
|
||||
private static SSLContext getSslContext(ZeppelinConfiguration conf)
|
||||
throws Exception {
|
||||
private static void setupRestApiContextHandler(WebAppContext webapp,
|
||||
ZeppelinConfiguration conf) {
|
||||
|
||||
SslContextFactory scf = getSslContextFactory(conf);
|
||||
if (!scf.isStarted()) {
|
||||
scf.start();
|
||||
}
|
||||
return scf.getSslContext();
|
||||
}
|
||||
|
||||
private static ServletContextHandler setupRestApiContextHandler(ZeppelinConfiguration conf) {
|
||||
final ServletHolder cxfServletHolder = new ServletHolder(new CXFNonSpringJaxrsServlet());
|
||||
cxfServletHolder.setInitParameter("javax.ws.rs.Application", ZeppelinServer.class.getName());
|
||||
cxfServletHolder.setName("rest");
|
||||
cxfServletHolder.setForcedPath("rest");
|
||||
|
||||
final ServletContextHandler cxfContext = new ServletContextHandler();
|
||||
cxfContext.setSessionHandler(new SessionHandler());
|
||||
cxfContext.setContextPath(conf.getServerContextPath());
|
||||
cxfContext.addServlet(cxfServletHolder, "/api/*");
|
||||
webapp.setSessionHandler(new SessionHandler());
|
||||
webapp.addServlet(cxfServletHolder, "/api/*");
|
||||
|
||||
cxfContext.addFilter(new FilterHolder(CorsFilter.class), "/*",
|
||||
webapp.addFilter(new FilterHolder(CorsFilter.class), "/*",
|
||||
EnumSet.allOf(DispatcherType.class));
|
||||
|
||||
cxfContext.setInitParameter("shiroConfigLocations",
|
||||
webapp.setInitParameter("shiroConfigLocations",
|
||||
new File(conf.getShiroPath()).toURI().toString());
|
||||
|
||||
cxfContext.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*",
|
||||
webapp.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/*",
|
||||
EnumSet.allOf(DispatcherType.class));
|
||||
|
||||
cxfContext.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener());
|
||||
webapp.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener());
|
||||
|
||||
return cxfContext;
|
||||
}
|
||||
|
||||
private static WebAppContext setupWebAppContext(
|
||||
private static WebAppContext setupWebAppContext(ContextHandlerCollection contexts,
|
||||
ZeppelinConfiguration conf) {
|
||||
|
||||
WebAppContext webApp = new WebAppContext();
|
||||
|
|
@ -282,7 +286,10 @@ public class ZeppelinServer extends Application {
|
|||
}
|
||||
// Explicit bind to root
|
||||
webApp.addServlet(new ServletHolder(new DefaultServlet()), "/*");
|
||||
contexts.addHandler(webApp);
|
||||
|
||||
return webApp;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
*/
|
||||
package org.apache.zeppelin.socket;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.UnknownHostException;
|
||||
|
|
@ -50,15 +49,14 @@ import org.apache.zeppelin.server.ZeppelinServer;
|
|||
import org.apache.zeppelin.socket.Message.OP;
|
||||
import org.apache.zeppelin.ticket.TicketContainer;
|
||||
import org.apache.zeppelin.utils.SecurityUtils;
|
||||
import org.eclipse.jetty.websocket.WebSocket;
|
||||
import org.eclipse.jetty.websocket.WebSocketServlet;
|
||||
import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
|
||||
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
|
||||
import org.quartz.SchedulerException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Zeppelin websocket service.
|
||||
*
|
||||
*/
|
||||
public class NotebookServer extends WebSocketServlet implements
|
||||
NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener,
|
||||
|
|
@ -73,6 +71,10 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
}
|
||||
|
||||
@Override
|
||||
public void configure(WebSocketServletFactory factory) {
|
||||
factory.setCreator(new NotebookWebSocketCreator(this));
|
||||
}
|
||||
|
||||
public boolean checkOrigin(HttpServletRequest request, String origin) {
|
||||
try {
|
||||
return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create());
|
||||
|
|
@ -84,8 +86,7 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public WebSocket doWebSocketConnect(HttpServletRequest req, String protocol) {
|
||||
public NotebookSocket doWebSocketConnect(HttpServletRequest req, String protocol) {
|
||||
return new NotebookSocket(req, protocol, this);
|
||||
}
|
||||
|
||||
|
|
@ -450,7 +451,7 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
}
|
||||
}
|
||||
|
||||
private void updateNote(WebSocket conn, HashSet<String> userAndRoles,
|
||||
private void updateNote(NotebookSocket conn, HashSet<String> userAndRoles,
|
||||
Notebook notebook, Message fromMessage)
|
||||
throws SchedulerException, IOException {
|
||||
String noteId = (String) fromMessage.get("id");
|
||||
|
|
@ -701,12 +702,12 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
List<InterpreterSetting> settings = note.getNoteReplLoader()
|
||||
.getInterpreterSettings();
|
||||
for (InterpreterSetting setting : settings) {
|
||||
if (setting.getInterpreterGroup() == null) {
|
||||
if (setting.getInterpreterGroup(n.id()) == null) {
|
||||
continue;
|
||||
}
|
||||
if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) {
|
||||
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) {
|
||||
AngularObjectRegistry angularObjectRegistry = setting
|
||||
.getInterpreterGroup().getAngularObjectRegistry();
|
||||
.getInterpreterGroup(n.id()).getAngularObjectRegistry();
|
||||
this.broadcastExcept(
|
||||
n.id(),
|
||||
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
|
||||
|
|
@ -1194,14 +1195,14 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
}
|
||||
|
||||
for (InterpreterSetting intpSetting : settings) {
|
||||
AngularObjectRegistry registry = intpSetting.getInterpreterGroup()
|
||||
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id())
|
||||
.getAngularObjectRegistry();
|
||||
List<AngularObject> objects = registry.getAllWithGlobal(note.id());
|
||||
for (AngularObject object : objects) {
|
||||
conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
|
||||
.put("angularObject", object)
|
||||
.put("interpreterGroupId",
|
||||
intpSetting.getInterpreterGroup().getId())
|
||||
intpSetting.getInterpreterGroup(note.id()).getId())
|
||||
.put("noteId", note.id())
|
||||
.put("paragraphId", object.getParagraphId())
|
||||
));
|
||||
|
|
|
|||
|
|
@ -20,19 +20,19 @@ import java.io.IOException;
|
|||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.eclipse.jetty.websocket.WebSocket;
|
||||
import org.eclipse.jetty.websocket.api.Session;
|
||||
import org.eclipse.jetty.websocket.api.WebSocketAdapter;
|
||||
|
||||
/**
|
||||
* Notebook websocket
|
||||
*/
|
||||
public class NotebookSocket implements WebSocket.OnTextMessage{
|
||||
public class NotebookSocket extends WebSocketAdapter {
|
||||
|
||||
private Connection connection;
|
||||
private Session connection;
|
||||
private NotebookSocketListener listener;
|
||||
private HttpServletRequest request;
|
||||
private String protocol;
|
||||
|
||||
|
||||
public NotebookSocket(HttpServletRequest req, String protocol,
|
||||
NotebookSocketListener listener) {
|
||||
this.listener = listener;
|
||||
|
|
@ -41,22 +41,22 @@ public class NotebookSocket implements WebSocket.OnTextMessage{
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onClose(int closeCode, String message) {
|
||||
public void onWebSocketClose(int closeCode, String message) {
|
||||
listener.onClose(this, closeCode, message);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOpen(Connection connection) {
|
||||
public void onWebSocketConnect(Session connection) {
|
||||
this.connection = connection;
|
||||
listener.onOpen(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onMessage(String message) {
|
||||
public void onWebSocketText(String message) {
|
||||
listener.onMessage(this, message);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public HttpServletRequest getRequest() {
|
||||
return request;
|
||||
}
|
||||
|
|
@ -66,8 +66,7 @@ public class NotebookSocket implements WebSocket.OnTextMessage{
|
|||
}
|
||||
|
||||
public void send(String serializeMessage) throws IOException {
|
||||
connection.sendMessage(serializeMessage);
|
||||
connection.getRemote().sendString(serializeMessage);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package org.apache.zeppelin.socket;
|
|||
* NoteboookSocket listener
|
||||
*/
|
||||
public interface NotebookSocketListener {
|
||||
public void onClose(NotebookSocket socket, int code, String message);
|
||||
public void onOpen(NotebookSocket socket);
|
||||
public void onMessage(NotebookSocket socket, String message);
|
||||
void onClose(NotebookSocket socket, int code, String message);
|
||||
void onOpen(NotebookSocket socket);
|
||||
void onMessage(NotebookSocket socket, String message);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.socket;
|
||||
|
||||
import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest;
|
||||
import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse;
|
||||
import org.eclipse.jetty.websocket.servlet.WebSocketCreator;
|
||||
|
||||
/**
|
||||
* Responsible to create the WebSockets for the NotebookServer.
|
||||
*/
|
||||
public class NotebookWebSocketCreator implements WebSocketCreator {
|
||||
private NotebookServer notebookServer;
|
||||
|
||||
public NotebookWebSocketCreator(NotebookServer notebookServer) {
|
||||
this.notebookServer = notebookServer;
|
||||
}
|
||||
public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) {
|
||||
return new NotebookSocket(request.getHttpServletRequest(), "", notebookServer);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -205,7 +205,8 @@ public class ZeppelinIT extends AbstractZeppelinIT {
|
|||
String artifact = "org.apache.commons:commons-csv:1.1";
|
||||
depArtifact.sendKeys(artifact);
|
||||
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
|
||||
driver.switchTo().alert().accept();
|
||||
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
|
||||
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
|
||||
|
||||
driver.navigate().back();
|
||||
createNewNote();
|
||||
|
|
@ -241,7 +242,8 @@ public class ZeppelinIT extends AbstractZeppelinIT {
|
|||
sleep(5000, true);
|
||||
testDepRemoveBtn.click();
|
||||
driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
|
||||
driver.switchTo().alert().accept();
|
||||
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" +
|
||||
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
|
||||
} catch (Exception e) {
|
||||
handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -374,13 +374,13 @@ public abstract class AbstractTestRestApi {
|
|||
//Create new Setting and return Setting ID
|
||||
protected String createTempSetting(String tempName)
|
||||
throws IOException, RepositoryException {
|
||||
InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory()
|
||||
InterpreterSetting setting = ZeppelinServer.notebook.getInterpreterFactory()
|
||||
.add(tempName,
|
||||
"newGroup",
|
||||
new LinkedList<Dependency>(),
|
||||
new InterpreterOption(false),
|
||||
new Properties());
|
||||
return interpreterGroup.getId();
|
||||
return setting.id();
|
||||
}
|
||||
|
||||
protected TypeSafeMatcher<? super JsonElement> hasRootElementNamed(final String memberName) {
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
|
|||
List<InterpreterSetting> settings = note1.getNoteReplLoader().getInterpreterSettings();
|
||||
for (InterpreterSetting setting : settings) {
|
||||
if (setting.getName().equals("md")) {
|
||||
interpreterGroup = setting.getInterpreterGroup();
|
||||
interpreterGroup = setting.getInterpreterGroup("sharedProcess");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,7 +434,7 @@ module.exports = function (grunt) {
|
|||
]);
|
||||
|
||||
grunt.registerTask('build', [
|
||||
'newer:jshint',
|
||||
'jshint:all',
|
||||
'clean:dist',
|
||||
'wiredep',
|
||||
'useminPrepare',
|
||||
|
|
|
|||
|
|
@ -124,7 +124,6 @@
|
|||
</goals>
|
||||
<configuration>
|
||||
<arguments>build</arguments>
|
||||
<arguments>--force</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
|
||||
|
|
|
|||
|
|
@ -36,9 +36,39 @@ limitations under the License.
|
|||
</div>
|
||||
|
||||
<b>Option</b>
|
||||
<div class="checkbox">
|
||||
<label><input type="checkbox" style="top:-5px" ng-model="newInterpreterSetting.option.perNoteSession">Separate Interpreter for each note</input></label>
|
||||
<div>
|
||||
<span class="btn-group">
|
||||
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
|
||||
data-toggle="dropdown">
|
||||
{{getSessionOption(setting.id)}} <span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu" role="menu">
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Single interpreter instance are shared across notes"
|
||||
ng-click="setSessionOption(setting.id, 'shared')">
|
||||
shared
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Separate Interpreter instance for each note"
|
||||
ng-click="setSessionOption(setting.id, 'scoped')">
|
||||
scoped
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Separate Interpreter process for each note"
|
||||
ng-click="setSessionOption(setting.id, 'isolated')">
|
||||
isolated
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
<span>Interpreter for note</span>
|
||||
</div>
|
||||
<br />
|
||||
|
||||
<b>Properties</b>
|
||||
<table class="table table-striped properties">
|
||||
|
|
|
|||
|
|
@ -60,44 +60,89 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
|
|||
interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]);
|
||||
};
|
||||
|
||||
$scope.updateInterpreterSetting = function(form, settingId) {
|
||||
var result = confirm('Do you want to update this interpreter and restart with new settings?');
|
||||
if (result) {
|
||||
$scope.setSessionOption = function(settingId, sessionOption) {
|
||||
var option;
|
||||
if (settingId === undefined) {
|
||||
option = $scope.newInterpreterSetting.option;
|
||||
} else {
|
||||
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
|
||||
var setting = $scope.interpreterSettings[index];
|
||||
if (setting.propertyKey !== '' || setting.propertyKey) {
|
||||
$scope.addNewInterpreterProperty(settingId);
|
||||
}
|
||||
if (setting.depArtifact !== '' || setting.depArtifact) {
|
||||
$scope.addNewInterpreterDependency(settingId);
|
||||
}
|
||||
|
||||
// add missing field of option
|
||||
if (!setting.option) {
|
||||
setting.option = {};
|
||||
}
|
||||
if (setting.option.remote === undefined) {
|
||||
// remote always true for now
|
||||
setting.option.remote = true;
|
||||
}
|
||||
|
||||
var request = {
|
||||
option: angular.copy(setting.option),
|
||||
properties: angular.copy(setting.properties),
|
||||
dependencies: angular.copy(setting.dependencies)
|
||||
};
|
||||
|
||||
$http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request).
|
||||
success(function (data, status, headers, config) {
|
||||
$scope.interpreterSettings[index] = data.body;
|
||||
removeTMPSettings(index);
|
||||
}).
|
||||
error(function (data, status, headers, config) {
|
||||
console.log('Error %o %o', status, data.message);
|
||||
ngToast.danger({content: data.message, verticalPosition: 'bottom'});
|
||||
form.$show();
|
||||
});
|
||||
option = setting.option;
|
||||
}
|
||||
|
||||
if (sessionOption === 'isolated') {
|
||||
option.perNoteSession = false;
|
||||
option.perNoteProcess = true;
|
||||
} else if (sessionOption === 'scoped') {
|
||||
option.perNoteSession = true;
|
||||
option.perNoteProcess = false;
|
||||
} else {
|
||||
option.perNoteSession = false;
|
||||
option.perNoteProcess = false;
|
||||
}
|
||||
};
|
||||
|
||||
$scope.getSessionOption = function(settingId) {
|
||||
var option;
|
||||
if (settingId === undefined) {
|
||||
option = $scope.newInterpreterSetting.option;
|
||||
} else {
|
||||
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
|
||||
var setting = $scope.interpreterSettings[index];
|
||||
option = setting.option;
|
||||
}
|
||||
|
||||
if (option.perNoteSession) {
|
||||
return 'scoped';
|
||||
} else if (option.perNoteProcess) {
|
||||
return 'isolated';
|
||||
} else {
|
||||
return 'shared';
|
||||
}
|
||||
};
|
||||
|
||||
$scope.updateInterpreterSetting = function(form, settingId) {
|
||||
BootstrapDialog.confirm({
|
||||
closable: true,
|
||||
title: '',
|
||||
message: 'Do you want to update this interpreter and restart with new settings?',
|
||||
callback: function (result) {
|
||||
if (result) {
|
||||
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
|
||||
var setting = $scope.interpreterSettings[index];
|
||||
if (setting.propertyKey !== '' || setting.propertyKey) {
|
||||
$scope.addNewInterpreterProperty(settingId);
|
||||
}
|
||||
if (setting.depArtifact !== '' || setting.depArtifact) {
|
||||
$scope.addNewInterpreterDependency(settingId);
|
||||
}
|
||||
// add missing field of option
|
||||
if (!setting.option) {
|
||||
setting.option = {};
|
||||
}
|
||||
if (setting.option.remote === undefined) {
|
||||
// remote always true for now
|
||||
setting.option.remote = true;
|
||||
}
|
||||
var request = {
|
||||
option: angular.copy(setting.option),
|
||||
properties: angular.copy(setting.properties),
|
||||
dependencies: angular.copy(setting.dependencies)
|
||||
};
|
||||
|
||||
$http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request).
|
||||
success(function (data, status, headers, config) {
|
||||
$scope.interpreterSettings[index] = data.body;
|
||||
removeTMPSettings(index);
|
||||
}).
|
||||
error(function (data, status, headers, config) {
|
||||
console.log('Error %o %o', status, data.message);
|
||||
ngToast.danger({content: data.message, verticalPosition: 'bottom'});
|
||||
form.$show();
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
$scope.resetInterpreterSetting = function(settingId){
|
||||
|
|
@ -227,7 +272,8 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope,
|
|||
dependencies: [],
|
||||
option: {
|
||||
remote: true,
|
||||
perNoteSession: false
|
||||
perNoteSession: false,
|
||||
perNoteProcess: false
|
||||
}
|
||||
};
|
||||
emptyNewProperty($scope.newInterpreterSetting);
|
||||
|
|
|
|||
|
|
@ -111,16 +111,39 @@ limitations under the License.
|
|||
<div class="row interpreter">
|
||||
<div class="col-md-12">
|
||||
<h5>Option</h5>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox"
|
||||
style="top:-5px"
|
||||
ng-disabled="!valueform.$visible"
|
||||
ng-model="setting.option.perNoteSession">
|
||||
Separate Interpreter for each note</input>
|
||||
</label>
|
||||
</div>
|
||||
<span class="btn-group">
|
||||
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
|
||||
data-toggle="dropdown"
|
||||
ng-disabled="!valueform.$visible">
|
||||
{{getSessionOption(setting.id)}} <span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu" role="menu">
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Single interpreter instance are shared across notes"
|
||||
ng-click="setSessionOption(setting.id, 'shared')">
|
||||
shared
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Separate Interpreter instance for each note"
|
||||
ng-click="setSessionOption(setting.id, 'scoped')">
|
||||
scoped
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a style="cursor:pointer"
|
||||
tooltip="Separate Interpreter process for each note"
|
||||
ng-click="setSessionOption(setting.id, 'isolated')">
|
||||
isolated
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
<span>Interpreter for note</span>
|
||||
</div>
|
||||
|
||||
|
||||
<div ng-show="_.isEmpty(setting.properties) && _.isEmpty(setting.dependencies) || valueform.$hidden" class="col-md-12 gray40-message">
|
||||
<em>Currently there are no properties and dependencies set for this interpreter</em>
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ limitations under the License.
|
|||
<script src="bower_components/angular-websocket/angular-websocket.min.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/ace.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/mode-scala.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/mode-python.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/mode-sql.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/mode-markdown.js"></script>
|
||||
<script src="bower_components/ace-builds/src-noconflict/mode-sh.js"></script>
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ module.exports = function(config) {
|
|||
'bower_components/angular-websocket/angular-websocket.min.js',
|
||||
'bower_components/ace-builds/src-noconflict/ace.js',
|
||||
'bower_components/ace-builds/src-noconflict/mode-scala.js',
|
||||
'bower_components/ace-builds/src-noconflict/mode-python.js',
|
||||
'bower_components/ace-builds/src-noconflict/mode-sql.js',
|
||||
'bower_components/ace-builds/src-noconflict/mode-markdown.js',
|
||||
'bower_components/ace-builds/src-noconflict/mode-sh.js',
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer;
|
|||
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
|
||||
import org.apache.zeppelin.notebook.NoteInterpreterLoader;
|
||||
import org.apache.zeppelin.scheduler.Job;
|
||||
import org.apache.zeppelin.scheduler.Job.Status;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -55,7 +56,7 @@ import java.util.*;
|
|||
/**
|
||||
* Manage interpreters.
|
||||
*/
|
||||
public class InterpreterFactory {
|
||||
public class InterpreterFactory implements InterpreterGroupFactory {
|
||||
Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
|
||||
|
||||
private Map<String, URLClassLoader> cleanCl = Collections
|
||||
|
|
@ -255,9 +256,7 @@ public class InterpreterFactory {
|
|||
setting.getDependencies(),
|
||||
setting.getOption());
|
||||
|
||||
InterpreterGroup interpreterGroup = createInterpreterGroup(setting.id(), setting.getOption());
|
||||
intpSetting.setInterpreterGroup(interpreterGroup);
|
||||
|
||||
intpSetting.setInterpreterGroupFactory(this);
|
||||
interpreterSettings.put(k, intpSetting);
|
||||
}
|
||||
|
||||
|
|
@ -381,7 +380,7 @@ public class InterpreterFactory {
|
|||
* @throws InterpreterException
|
||||
* @throws IOException
|
||||
*/
|
||||
public InterpreterGroup add(String name, String groupName,
|
||||
public InterpreterSetting add(String name, String groupName,
|
||||
List<Dependency> dependencies,
|
||||
InterpreterOption option, Properties properties)
|
||||
throws InterpreterException, IOException, RepositoryException {
|
||||
|
|
@ -415,17 +414,15 @@ public class InterpreterFactory {
|
|||
loadInterpreterDependencies(intpSetting);
|
||||
}
|
||||
|
||||
InterpreterGroup interpreterGroup = createInterpreterGroup(intpSetting.id(), option);
|
||||
|
||||
intpSetting.setInterpreterGroup(interpreterGroup);
|
||||
|
||||
intpSetting.setInterpreterGroupFactory(this);
|
||||
interpreterSettings.put(intpSetting.id(), intpSetting);
|
||||
saveToFile();
|
||||
return interpreterGroup;
|
||||
return intpSetting;
|
||||
}
|
||||
}
|
||||
|
||||
private InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
|
||||
@Override
|
||||
public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
|
||||
throws InterpreterException, NullArgumentException {
|
||||
|
||||
//When called from REST API without option we receive NPE
|
||||
|
|
@ -455,25 +452,28 @@ public class InterpreterFactory {
|
|||
|
||||
public void removeInterpretersForNote(InterpreterSetting interpreterSetting,
|
||||
String noteId) {
|
||||
if (!interpreterSetting.getOption().isPerNoteSession()) {
|
||||
return;
|
||||
if (interpreterSetting.getOption().isPerNoteProcess()) {
|
||||
interpreterSetting.closeAndRemoveInterpreterGroup(noteId);
|
||||
} else if (interpreterSetting.getOption().isPerNoteSession()) {
|
||||
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId);
|
||||
|
||||
interpreterGroup.close(noteId);
|
||||
interpreterGroup.destroy(noteId);
|
||||
synchronized (interpreterGroup) {
|
||||
interpreterGroup.remove(noteId);
|
||||
interpreterGroup.notifyAll(); // notify createInterpreterForNote()
|
||||
}
|
||||
logger.info("Interpreter instance {} for note {} is removed",
|
||||
interpreterSetting.getName(),
|
||||
noteId);
|
||||
}
|
||||
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup();
|
||||
interpreterGroup.close(noteId);
|
||||
interpreterGroup.destroy(noteId);
|
||||
synchronized (interpreterGroup) {
|
||||
interpreterGroup.remove(noteId);
|
||||
interpreterGroup.notifyAll(); // notify createInterpreterForNote()
|
||||
}
|
||||
logger.info("Interpreter instance {} for note {} is removed",
|
||||
interpreterSetting.getName(),
|
||||
noteId);
|
||||
}
|
||||
|
||||
public void createInterpretersForNote(
|
||||
InterpreterSetting interpreterSetting,
|
||||
String noteId) {
|
||||
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup();
|
||||
String noteId,
|
||||
String key) {
|
||||
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId);
|
||||
String groupName = interpreterSetting.getGroup();
|
||||
InterpreterOption option = interpreterSetting.getOption();
|
||||
Properties properties = interpreterSetting.getProperties();
|
||||
|
|
@ -484,10 +484,12 @@ public class InterpreterFactory {
|
|||
// interpreter process supposed to be terminated by RemoteInterpreterProcess.dereference()
|
||||
// in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and
|
||||
// removal from interpreter group take too long, throw an error.
|
||||
long minTimeout = 10 * 1000 * 1000000; // 10 sec
|
||||
long minTimeout = 10L * 1000 * 1000000; // 10 sec
|
||||
long interpreterRemovalWaitTimeout =
|
||||
Math.max(minTimeout, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 2);
|
||||
while (interpreterGroup.containsKey(noteId)) {
|
||||
Math.max(
|
||||
minTimeout,
|
||||
conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2);
|
||||
while (interpreterGroup.containsKey(key)) {
|
||||
if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) {
|
||||
throw new InterpreterException("Can not create interpreter");
|
||||
}
|
||||
|
|
@ -517,10 +519,10 @@ public class InterpreterFactory {
|
|||
option.getHost(), option.getPort(), properties);
|
||||
} else {
|
||||
intp = createRemoteRepl(info.getPath(),
|
||||
noteId,
|
||||
key,
|
||||
info.getClassName(),
|
||||
properties,
|
||||
interpreterGroup.id);
|
||||
interpreterSetting.id());
|
||||
}
|
||||
} else {
|
||||
intp = createRepl(info.getPath(),
|
||||
|
|
@ -529,10 +531,10 @@ public class InterpreterFactory {
|
|||
}
|
||||
|
||||
synchronized (interpreterGroup) {
|
||||
List<Interpreter> interpreters = interpreterGroup.get(noteId);
|
||||
List<Interpreter> interpreters = interpreterGroup.get(key);
|
||||
if (interpreters == null) {
|
||||
interpreters = new LinkedList<Interpreter>();
|
||||
interpreterGroup.put(noteId, interpreters);
|
||||
interpreterGroup.put(key, interpreters);
|
||||
}
|
||||
interpreters.add(intp);
|
||||
}
|
||||
|
|
@ -550,8 +552,7 @@ public class InterpreterFactory {
|
|||
synchronized (interpreterSettings) {
|
||||
if (interpreterSettings.containsKey(id)) {
|
||||
InterpreterSetting intp = interpreterSettings.get(id);
|
||||
intp.getInterpreterGroup().close();
|
||||
intp.getInterpreterGroup().destroy();
|
||||
intp.closeAndRmoveAllInterpreterGroups();
|
||||
|
||||
interpreterSettings.remove(id);
|
||||
for (List<String> settings : interpreterBindings.values()) {
|
||||
|
|
@ -681,16 +682,12 @@ public class InterpreterFactory {
|
|||
|
||||
stopJobAllInterpreter(intpsetting);
|
||||
|
||||
intpsetting.getInterpreterGroup().close();
|
||||
intpsetting.getInterpreterGroup().destroy();
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
|
||||
intpsetting.setOption(option);
|
||||
intpsetting.setProperties(properties);
|
||||
intpsetting.setDependencies(dependencies);
|
||||
|
||||
InterpreterGroup interpreterGroup = createInterpreterGroup(intpsetting.id(), option);
|
||||
intpsetting.setInterpreterGroup(interpreterGroup);
|
||||
|
||||
loadInterpreterDependencies(intpsetting);
|
||||
saveToFile();
|
||||
} else {
|
||||
|
|
@ -707,13 +704,8 @@ public class InterpreterFactory {
|
|||
|
||||
stopJobAllInterpreter(intpsetting);
|
||||
|
||||
intpsetting.getInterpreterGroup().close();
|
||||
intpsetting.getInterpreterGroup().destroy();
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
|
||||
InterpreterGroup interpreterGroup = createInterpreterGroup(
|
||||
intpsetting.id(),
|
||||
intpsetting.getOption());
|
||||
intpsetting.setInterpreterGroup(interpreterGroup);
|
||||
} else {
|
||||
throw new InterpreterException("Interpreter setting id " + id
|
||||
+ " not found");
|
||||
|
|
@ -723,17 +715,19 @@ public class InterpreterFactory {
|
|||
|
||||
private void stopJobAllInterpreter(InterpreterSetting intpsetting) {
|
||||
if (intpsetting != null) {
|
||||
for (List<Interpreter> interpreters : intpsetting.getInterpreterGroup().values()) {
|
||||
for (Interpreter intp : interpreters) {
|
||||
for (Job job : intp.getScheduler().getJobsRunning()) {
|
||||
job.abort();
|
||||
job.setStatus(Status.ABORT);
|
||||
logger.info("Job " + job.getJobName() + " aborted ");
|
||||
}
|
||||
for (Job job : intp.getScheduler().getJobsWaiting()) {
|
||||
job.abort();
|
||||
job.setStatus(Status.ABORT);
|
||||
logger.info("Job " + job.getJobName() + " aborted ");
|
||||
for (InterpreterGroup intpGroup : intpsetting.getAllInterpreterGroups()) {
|
||||
for (List<Interpreter> interpreters : intpGroup.values()) {
|
||||
for (Interpreter intp : interpreters) {
|
||||
for (Job job : intp.getScheduler().getJobsRunning()) {
|
||||
job.abort();
|
||||
job.setStatus(Status.ABORT);
|
||||
logger.info("Job " + job.getJobName() + " aborted ");
|
||||
}
|
||||
for (Job job : intp.getScheduler().getJobsWaiting()) {
|
||||
job.abort();
|
||||
job.setStatus(Status.ABORT);
|
||||
logger.info("Job " + job.getJobName() + " aborted ");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -747,8 +741,7 @@ public class InterpreterFactory {
|
|||
for (final InterpreterSetting intpsetting : intpsettings) {
|
||||
Thread t = new Thread() {
|
||||
public void run() {
|
||||
intpsetting.getInterpreterGroup().close();
|
||||
intpsetting.getInterpreterGroup().destroy();
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
}
|
||||
};
|
||||
t.start();
|
||||
|
|
@ -847,9 +840,9 @@ public class InterpreterFactory {
|
|||
}
|
||||
|
||||
private Interpreter createRemoteRepl(String interpreterPath, String noteId, String className,
|
||||
Properties property, String interpreterId) {
|
||||
Properties property, String interpreterSettingId) {
|
||||
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
|
||||
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterId;
|
||||
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
|
||||
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
|
||||
RemoteInterpreter remoteInterpreter = new RemoteInterpreter(
|
||||
property, noteId, className, conf.getInterpreterRemoteRunnerPath(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.interpreter;
|
||||
|
||||
import org.apache.commons.lang.NullArgumentException;
|
||||
|
||||
/**
|
||||
* Created InterpreterGroup
|
||||
*/
|
||||
public interface InterpreterGroupFactory {
|
||||
InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option);
|
||||
}
|
||||
|
|
@ -25,6 +25,7 @@ public class InterpreterOption {
|
|||
String host = null;
|
||||
int port = -1;
|
||||
boolean perNoteSession;
|
||||
boolean perNoteProcess;
|
||||
|
||||
public InterpreterOption() {
|
||||
remote = false;
|
||||
|
|
@ -50,7 +51,6 @@ public class InterpreterOption {
|
|||
this.perNoteSession = perNoteSession;
|
||||
}
|
||||
|
||||
|
||||
public boolean isConnectExistingProcess() {
|
||||
return (host != null && port != -1);
|
||||
}
|
||||
|
|
@ -62,4 +62,12 @@ public class InterpreterOption {
|
|||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public boolean isPerNoteProcess() {
|
||||
return perNoteProcess;
|
||||
}
|
||||
|
||||
public void setPerNoteProcess(boolean perNoteProcess) {
|
||||
this.perNoteProcess = perNoteProcess;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,28 +17,32 @@
|
|||
|
||||
package org.apache.zeppelin.interpreter;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Random;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.apache.commons.lang.NullArgumentException;
|
||||
import org.apache.zeppelin.dep.Dependency;
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
|
||||
import org.apache.zeppelin.notebook.utility.IdHashes;
|
||||
|
||||
/**
|
||||
* Interpreter settings
|
||||
*/
|
||||
public class InterpreterSetting {
|
||||
private static final String SHARED_PROCESS = "shared_process";
|
||||
private String id;
|
||||
private String name;
|
||||
private String group;
|
||||
private String description;
|
||||
private Properties properties;
|
||||
private transient InterpreterGroupFactory interpreterGroupFactory;
|
||||
|
||||
// use 'interpreterGroup' as a field name to keep backward compatibility of
|
||||
// conf/interpreter.json file format
|
||||
private List<InterpreterInfo> interpreterGroup;
|
||||
private transient InterpreterGroup interpreterGroupRef;
|
||||
private transient Map<String, InterpreterGroup> interpreterGroupRef =
|
||||
new HashMap<String, InterpreterGroup>();
|
||||
private List<Dependency> dependencies;
|
||||
private InterpreterOption option;
|
||||
|
||||
|
|
@ -56,6 +60,7 @@ public class InterpreterSetting {
|
|||
this.properties = properties;
|
||||
this.dependencies = dependencies;
|
||||
this.option = option;
|
||||
this.interpreterGroupFactory = interpreterGroupFactory;
|
||||
}
|
||||
|
||||
public InterpreterSetting(String name,
|
||||
|
|
@ -117,12 +122,54 @@ public class InterpreterSetting {
|
|||
return group;
|
||||
}
|
||||
|
||||
public InterpreterGroup getInterpreterGroup() {
|
||||
return interpreterGroupRef;
|
||||
|
||||
private String getInterpreterProcessKey(String noteId) {
|
||||
if (getOption().isPerNoteProcess()) {
|
||||
return noteId;
|
||||
} else {
|
||||
return SHARED_PROCESS;
|
||||
}
|
||||
}
|
||||
|
||||
public void setInterpreterGroup(InterpreterGroup interpreterGroup) {
|
||||
this.interpreterGroupRef = interpreterGroup;
|
||||
public InterpreterGroup getInterpreterGroup(String noteId) {
|
||||
String key = getInterpreterProcessKey(noteId);
|
||||
synchronized (interpreterGroupRef) {
|
||||
if (!interpreterGroupRef.containsKey(key)) {
|
||||
String interpreterGroupId = id() + ":" + key;
|
||||
InterpreterGroup intpGroup =
|
||||
interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
|
||||
interpreterGroupRef.put(key, intpGroup);
|
||||
}
|
||||
return interpreterGroupRef.get(key);
|
||||
}
|
||||
}
|
||||
|
||||
public Collection<InterpreterGroup> getAllInterpreterGroups() {
|
||||
synchronized (interpreterGroupRef) {
|
||||
return new LinkedList<InterpreterGroup>(interpreterGroupRef.values());
|
||||
}
|
||||
}
|
||||
|
||||
public void closeAndRemoveInterpreterGroup(String noteId) {
|
||||
String key = getInterpreterProcessKey(noteId);
|
||||
InterpreterGroup groupToRemove;
|
||||
synchronized (interpreterGroupRef) {
|
||||
groupToRemove = interpreterGroupRef.remove(key);
|
||||
}
|
||||
|
||||
if (groupToRemove != null) {
|
||||
groupToRemove.close();
|
||||
groupToRemove.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
public void closeAndRmoveAllInterpreterGroups() {
|
||||
synchronized (interpreterGroupRef) {
|
||||
HashSet<String> groupsToRemove = new HashSet<String>(interpreterGroupRef.keySet());
|
||||
for (String key : groupsToRemove) {
|
||||
closeAndRemoveInterpreterGroup(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Properties getProperties() {
|
||||
|
|
@ -159,4 +206,12 @@ public class InterpreterSetting {
|
|||
public List<InterpreterInfo> getInterpreterInfos() {
|
||||
return interpreterGroup;
|
||||
}
|
||||
|
||||
public InterpreterGroupFactory getInterpreterGroupFactory() {
|
||||
return interpreterGroupFactory;
|
||||
}
|
||||
|
||||
public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
|
||||
this.interpreterGroupFactory = interpreterGroupFactory;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -419,7 +419,7 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
}
|
||||
|
||||
for (InterpreterSetting setting : settings) {
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup();
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup(id);
|
||||
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
|
||||
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
|
||||
}
|
||||
|
|
@ -434,7 +434,7 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
}
|
||||
|
||||
for (InterpreterSetting setting : settings) {
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup();
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup(id);
|
||||
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
|
||||
|
||||
if (registry instanceof RemoteAngularObjectRegistry) {
|
||||
|
|
|
|||
|
|
@ -75,22 +75,23 @@ public class NoteInterpreterLoader {
|
|||
return settings;
|
||||
}
|
||||
|
||||
private String getInterpreterGroupKey(InterpreterSetting setting) {
|
||||
if (!setting.getOption().isPerNoteSession()) {
|
||||
return SHARED_SESSION;
|
||||
} else {
|
||||
private String getInterpreterInstanceKey(InterpreterSetting setting) {
|
||||
if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) {
|
||||
return noteId;
|
||||
} else {
|
||||
return SHARED_SESSION;
|
||||
}
|
||||
}
|
||||
|
||||
private List<Interpreter> createOrGetInterpreterList(InterpreterSetting setting) {
|
||||
InterpreterGroup interpreterGroup = setting.getInterpreterGroup();
|
||||
InterpreterGroup interpreterGroup =
|
||||
setting.getInterpreterGroup(noteId);
|
||||
synchronized (interpreterGroup) {
|
||||
String key = getInterpreterGroupKey(setting);
|
||||
String key = getInterpreterInstanceKey(setting);
|
||||
if (!interpreterGroup.containsKey(key)) {
|
||||
factory.createInterpretersForNote(setting, key);
|
||||
factory.createInterpretersForNote(setting, noteId, key);
|
||||
}
|
||||
return interpreterGroup.get(getInterpreterGroupKey(setting));
|
||||
return interpreterGroup.get(getInterpreterInstanceKey(setting));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -101,6 +102,7 @@ public class NoteInterpreterLoader {
|
|||
return;
|
||||
}
|
||||
|
||||
System.err.println("close");
|
||||
for (InterpreterSetting setting : settings) {
|
||||
factory.removeInterpretersForNote(setting, noteId);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -301,7 +301,7 @@ public class Notebook implements NoteEventListener {
|
|||
|
||||
// remove from all interpreter instance's angular object registry
|
||||
for (InterpreterSetting settings : replFactory.get()) {
|
||||
AngularObjectRegistry registry = settings.getInterpreterGroup().getAngularObjectRegistry();
|
||||
AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
|
||||
if (registry instanceof RemoteAngularObjectRegistry) {
|
||||
// remove paragraph scope object
|
||||
for (Paragraph p : note.getParagraphs()) {
|
||||
|
|
@ -412,7 +412,7 @@ public class Notebook implements NoteEventListener {
|
|||
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
|
||||
List<InterpreterSetting> settings = replFactory.get();
|
||||
for (InterpreterSetting setting : settings) {
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup();
|
||||
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.id());
|
||||
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
|
||||
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
|
||||
String noteId = snapshot.getAngularObject().getNoteId();
|
||||
|
|
|
|||
|
|
@ -116,6 +116,7 @@ public class NotebookAuthorization {
|
|||
if (existingEntities == null) {
|
||||
noteAuthInfo.put("owners", new LinkedHashSet(entities));
|
||||
} else {
|
||||
existingEntities.clear();
|
||||
existingEntities.addAll(entities);
|
||||
}
|
||||
}
|
||||
|
|
@ -135,6 +136,7 @@ public class NotebookAuthorization {
|
|||
if (existingEntities == null) {
|
||||
noteAuthInfo.put("readers", new LinkedHashSet(entities));
|
||||
} else {
|
||||
existingEntities.clear();
|
||||
existingEntities.addAll(entities);
|
||||
}
|
||||
}
|
||||
|
|
@ -154,6 +156,7 @@ public class NotebookAuthorization {
|
|||
if (existingEntities == null) {
|
||||
noteAuthInfo.put("writers", new LinkedHashSet(entities));
|
||||
} else {
|
||||
existingEntities.clear();
|
||||
existingEntities.addAll(entities);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -360,8 +360,8 @@ public class Paragraph extends Job implements Serializable, Cloneable {
|
|||
|
||||
if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
|
||||
InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
|
||||
registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
|
||||
resourcePool = intpGroup.getInterpreterGroup().getResourcePool();
|
||||
registry = intpGroup.getInterpreterGroup(note.id()).getAngularObjectRegistry();
|
||||
resourcePool = intpGroup.getInterpreterGroup(note.id()).getResourcePool();
|
||||
}
|
||||
|
||||
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
|
||||
|
|
|
|||
|
|
@ -75,8 +75,8 @@ public class InterpreterFactoryTest {
|
|||
public void testBasic() {
|
||||
List<String> all = factory.getDefaultInterpreterSettingList();
|
||||
InterpreterSetting setting = factory.get(all.get(0));
|
||||
InterpreterGroup interpreterGroup = setting.getInterpreterGroup();
|
||||
factory.createInterpretersForNote(setting, "session");
|
||||
InterpreterGroup interpreterGroup = setting.getInterpreterGroup("sharedProcess");
|
||||
factory.createInterpretersForNote(setting, "sharedProcess", "session");
|
||||
|
||||
// get interpreter
|
||||
Interpreter repl1 = interpreterGroup.get("session").get(0);
|
||||
|
|
@ -89,7 +89,7 @@ public class InterpreterFactoryTest {
|
|||
|
||||
// restart interpreter
|
||||
factory.restart(all.get(0));
|
||||
assertNull(setting.getInterpreterGroup().get("session"));
|
||||
assertNull(setting.getInterpreterGroup("sharedProcess").get("session"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -109,28 +109,68 @@ public class NoteInterpreterLoaderTest {
|
|||
loaderB.getInterpreterSettings().get(0).getOption().setPerNoteSession(true);
|
||||
|
||||
// interpreters are not created before accessing it
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup().get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup().get("noteB"));
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB"));
|
||||
|
||||
loaderA.get(null).open();
|
||||
loaderB.get(null).open();
|
||||
|
||||
// per note session interpreter instance in the same interpreter process
|
||||
assertTrue(
|
||||
loaderA.get(null).getInterpreterGroup().getRemoteInterpreterProcess() ==
|
||||
loaderB.get(null).getInterpreterGroup().getRemoteInterpreterProcess());
|
||||
loaderA.get(null).getInterpreterGroup().getId().equals(
|
||||
loaderB.get(null).getInterpreterGroup().getId()));
|
||||
|
||||
// interpreters are created after accessing it
|
||||
assertNotNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup().get("noteA"));
|
||||
assertNotNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup().get("noteB"));
|
||||
assertNotNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA"));
|
||||
assertNotNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB"));
|
||||
|
||||
// when
|
||||
loaderA.close();
|
||||
loaderB.close();
|
||||
|
||||
// interpreters are destroyed after close
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup().get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup().get("noteB"));
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB"));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNotePerInterpreterProcess() throws IOException {
|
||||
NoteInterpreterLoader loaderA = new NoteInterpreterLoader(factory);
|
||||
loaderA.setNoteId("noteA");
|
||||
loaderA.setInterpreters(factory.getDefaultInterpreterSettingList());
|
||||
loaderA.getInterpreterSettings().get(0).getOption().setPerNoteProcess(true);
|
||||
|
||||
NoteInterpreterLoader loaderB = new NoteInterpreterLoader(factory);
|
||||
loaderB.setNoteId("noteB");
|
||||
loaderB.setInterpreters(factory.getDefaultInterpreterSettingList());
|
||||
loaderB.getInterpreterSettings().get(0).getOption().setPerNoteProcess(true);
|
||||
|
||||
// interpreters are not created before accessing it
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB"));
|
||||
|
||||
loaderA.get(null).open();
|
||||
loaderB.get(null).open();
|
||||
|
||||
// per note interpreter process
|
||||
assertFalse(
|
||||
loaderA.get(null).getInterpreterGroup().getId().equals(
|
||||
loaderB.get(null).getInterpreterGroup().getId()));
|
||||
|
||||
// interpreters are created after accessing it
|
||||
assertNotNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA"));
|
||||
assertNotNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB"));
|
||||
|
||||
// when
|
||||
loaderA.close();
|
||||
loaderB.close();
|
||||
|
||||
// interpreters are destroyed after close
|
||||
assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA"));
|
||||
assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB"));
|
||||
}
|
||||
|
||||
|
||||
private void delete(File file){
|
||||
if(file.isFile()) file.delete();
|
||||
else if(file.isDirectory()){
|
||||
|
|
|
|||
|
|
@ -365,7 +365,7 @@ public class NotebookTest implements JobListenerFactory{
|
|||
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
|
||||
|
||||
AngularObjectRegistry registry = note.getNoteReplLoader()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess")
|
||||
.getAngularObjectRegistry();
|
||||
|
||||
Paragraph p1 = note.addParagraph();
|
||||
|
|
@ -398,7 +398,7 @@ public class NotebookTest implements JobListenerFactory{
|
|||
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
|
||||
|
||||
AngularObjectRegistry registry = note.getNoteReplLoader()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess")
|
||||
.getAngularObjectRegistry();
|
||||
|
||||
Paragraph p1 = note.addParagraph();
|
||||
|
|
@ -431,7 +431,7 @@ public class NotebookTest implements JobListenerFactory{
|
|||
note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList());
|
||||
|
||||
AngularObjectRegistry registry = note.getNoteReplLoader()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess")
|
||||
.getAngularObjectRegistry();
|
||||
|
||||
// add local scope object
|
||||
|
|
@ -442,7 +442,7 @@ public class NotebookTest implements JobListenerFactory{
|
|||
// restart interpreter
|
||||
factory.restart(note.getNoteReplLoader().getInterpreterSettings().get(0).id());
|
||||
registry = note.getNoteReplLoader()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup()
|
||||
.getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess")
|
||||
.getAngularObjectRegistry();
|
||||
|
||||
// local and global scope object should be removed
|
||||
|
|
|
|||
Loading…
Reference in a new issue