mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
mfelgamal
GitHub
commit
a4bcc0d7d6
56 changed files with 1497 additions and 215 deletions
|
|
@ -1,4 +1,4 @@
|
|||
#Zeppelin
|
||||
# Apache Zeppelin
|
||||
|
||||
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
|
||||
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ user3 = password4, role2
|
|||
### A sample for configuring Active Directory Realm
|
||||
#activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
|
||||
#activeDirectoryRealm.systemUsername = userNameA
|
||||
|
||||
#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html
|
||||
#activeDirectoryRealm.systemPassword = passwordA
|
||||
#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks
|
||||
#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
||||
#activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.group.order</name>
|
||||
<value>"spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase</value>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery</value>
|
||||
<description></description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
94
docs/CONTRIBUTING.md
Normal file
94
docs/CONTRIBUTING.md
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
# Contributing to Apache Zeppelin Documentation
|
||||
|
||||
## Folder Structure
|
||||
`docs/` folder is organized as below:
|
||||
|
||||
```
|
||||
docs/
|
||||
├── _includes/themes/zeppelin
|
||||
│ ├── _navigation.html
|
||||
│ └── default.html
|
||||
├── _layouts
|
||||
├── _plugins
|
||||
├── assets/themes/zeppelin -> {ASSET_PATH}
|
||||
│ ├── bootstrap
|
||||
│ ├── css
|
||||
│ ├── img
|
||||
│ └── js
|
||||
├── development/ *.md
|
||||
├── displaysystem/ *.md
|
||||
├── install/ *.md
|
||||
├── interpreter/ *.md
|
||||
├── manual/ *.md
|
||||
├── quickstart/ *.md
|
||||
├── rest-api/ *.md
|
||||
├── security/ *.md
|
||||
├── storage/ *.md
|
||||
├── Gemfile
|
||||
├── Gemfile.lock
|
||||
├── _config.yml
|
||||
├── index.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
- `_navigation.html`: the dropdown menu in navbar
|
||||
- `default.html` & `_layouts/`: define default HTML layout
|
||||
- `_plugins/`: custom plugin `*.rb` files can be placed in this folder. See [jekyll/plugins](https://jekyllrb.com/docs/plugins/) for the further information.
|
||||
- `{ASSET_PATH}/css/style.css`: extra css components can be defined
|
||||
- `{ASSET_PATH}/img/docs-img/`: image files used for document pages can be placed in this folder
|
||||
- `{ASSET_PATH}/js/`: extra `.js` files can be placed
|
||||
- `Gemfile`: defines bundle dependencies. They will be installed by `bundle install`.
|
||||
- `Gemfile.lock`: when you run `bundle install`, bundler will persist all gems name and their version to this file. For the more details, see [Bundle "The Gemfile Lock"](http://bundler.io/v1.10/man/bundle-install.1.html#THE-GEMFILE-LOCK)
|
||||
- `documentation_group`: `development/`, `displaysystem/`, `install/`, `interpreter/`...
|
||||
- `_config.yml`: defines configuration options for docs website. See [jekyll/configuration](https://jekyllrb.com/docs/configuration/) for the other available config variables.
|
||||
- `index.md`: the main page of `http://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>/`
|
||||
|
||||
|
||||
## Markdown
|
||||
Zeppelin documentation pages are written with [Markdown](http://daringfireball.net/projects/markdown/). It is possible to use [GitHub flavored syntax](https://help.github.com/categories/writing-on-github/) and intermix plain HTML.
|
||||
|
||||
## Front matter
|
||||
Every page contains [YAML front matter](https://jekyllrb.com/docs/frontmatter/) block in their header. Don't forget to wrap the front matter list with triple-dashed lines(`---`) like below.
|
||||
The document page should start this triple-dashed lines. Or you will face 404 error, since Jekyll can't find the page.
|
||||
|
||||
```
|
||||
---
|
||||
layout: page
|
||||
title: "Apache Zeppelin Tutorial"
|
||||
description: "This tutorial page contains a short walk-through tutorial that uses Apache Spark backend. Please note that this tutorial is valid for Spark 1.3 and higher."
|
||||
group: quickstart
|
||||
---
|
||||
```
|
||||
|
||||
- `layout`: the default layout is `page` which is defined in `_layout/page.html`.
|
||||
- `title`: the title for the document. Please note that if it needs to include `Zeppelin`, it should be `Apache Zeppelin`, not `Zeppelin`.
|
||||
- `description`: a short description for the document. One or two sentences would be enough. This description also will be shown as an extract sentence when people search pages.
|
||||
- `group`: a category of the document page
|
||||
|
||||
## Headings
|
||||
All documents are structured with headings. From these headings, you can automatically generate a **Table of Contents**. There is a simple rule for Zeppelin docs headings.
|
||||
|
||||
```
|
||||
# Level-1 heading <- used only for the main title
|
||||
## Level-2 heading <- start with this
|
||||
### Level-3 heading
|
||||
#### Level-4 heading <- won't be converted in TOC from this level
|
||||
```
|
||||
|
||||
## Table of contents(TOC)
|
||||
|
||||
```
|
||||
<div id="toc"></div>
|
||||
```
|
||||
|
||||
Add this line below `# main title` in order to generate a **Table of Contents**. Headings until `### (Level-3 heading)` are included to TOC.
|
||||
|
||||
|
||||
Default setting options for TOC are definded in [here](https://github.com/apache/zeppelin/blob/master/docs/assets/themes/zeppelin/js/toc.js#L4).
|
||||
|
||||
|
||||
## Adding new pages
|
||||
If you're going to create new pages, there are some spots you need to add the location of the page.
|
||||
|
||||
- **Dropdown menu in navbar**: add your docs location to [_navigation.html](https://github.com/apache/zeppelin/blob/master/docs/_includes/themes/zeppelin/_navigation.html)
|
||||
- **Main index**: add your docs below [What is the next?](http://zeppelin.apache.org/docs/latest/#what-is-the-next) section in [index.md](https://github.com/apache/zeppelin/blob/master/docs/index.md) with a short description. No need to do this if the page is for **Interpreters**.
|
||||
|
|
@ -1,41 +1,55 @@
|
|||
## Apache Zeppelin documentation
|
||||
|
||||
This readme will walk you through building the Zeppelin documentation, which is included here with the Zeppelin source code.
|
||||
# Apache Zeppelin documentation
|
||||
|
||||
This README will walk you through building the documentation of Apache Zeppelin. The documentation is included here with Apache Zeppelin source code. The online documentation at [https://zeppelin.apache.org/docs/<ZEPPELIN_VERSION>](https://zeppelin.apache.org/docs/latest) is also generated from the files found in here.
|
||||
|
||||
## Build documentation
|
||||
See https://help.github.com/articles/using-jekyll-with-pages#installing-jekyll
|
||||
Zeppelin is using [Jekyll](https://jekyllrb.com/) which is a static site generator and [Github Pages](https://pages.github.com/) as a site publisher. For the more details, see [help.github.com/articles/about-github-pages-and-jekyll/](https://help.github.com/articles/about-github-pages-and-jekyll/).
|
||||
|
||||
**Requirements**
|
||||
|
||||
```
|
||||
ruby --version >= 2.0.0
|
||||
gem install bundler
|
||||
# go to /docs under your Zeppelin source
|
||||
bundle install
|
||||
# ruby --version >= 2.0.0
|
||||
# Install Bundler using gem
|
||||
gem install bundler
|
||||
|
||||
cd $ZEPPELIN_HOME/docs
|
||||
# Install all dependencies declared in the Gemfile
|
||||
bundle install
|
||||
```
|
||||
|
||||
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
|
||||
For the further information about requirements, please see [here](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements).
|
||||
|
||||
*On OS X 10.9 you may need to do "xcode-select --install"*
|
||||
On OS X 10.9, you may need to do
|
||||
|
||||
```
|
||||
xcode-select --install
|
||||
```
|
||||
|
||||
## Run website locally
|
||||
If you don't want to encounter uglily rendered pages, run the documentation site in your local first.
|
||||
|
||||
In `$ZEPPELIN_HOME/docs`,
|
||||
|
||||
```
|
||||
bundle exec jekyll serve --watch
|
||||
```
|
||||
|
||||
Using the above command, Jekyll will start a web server at `http://localhost:4000` and watch the `/docs` directory to update.
|
||||
|
||||
|
||||
## Run website
|
||||
|
||||
bundle exec jekyll serve --watch
|
||||
## Contribute to Zeppelin documentation
|
||||
If you wish to help us and contribute to Zeppelin Documentation, please look at [Zeppelin Documentation's contribution guideline](https://github.com/apache/zeppelin/blob/master/docs/CONTRIBUTING.md).
|
||||
|
||||
|
||||
## Adding a new page
|
||||
|
||||
rake page name="new-page.md"
|
||||
|
||||
|
||||
## Bumping up version in a new release
|
||||
## For committers only
|
||||
### Bumping up version in a new release
|
||||
|
||||
* `ZEPPELIN_VERSION` and `BASE_PATH` property in _config.yml
|
||||
|
||||
## Deploy to ASF svnpubsub infra (for committers only)
|
||||
### Deploy to ASF svnpubsub infra
|
||||
1. generate static website in `./_site`
|
||||
|
||||
```
|
||||
# go to /docs under Zeppelin source
|
||||
bundle exec jekyll build --safe
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@
|
|||
<li class="title"><span><b>Advanced</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (Standalone)</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/spark_cluster_mode.html#spark-standalone-mode">Zeppelin on Spark Cluster Mode (YARN)</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Contibute</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 245 KiB After Width: | Height: | Size: 195 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 95 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 212 KiB |
|
|
@ -170,6 +170,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
|
|||
* Advanced
|
||||
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
|
||||
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
|
||||
* [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-yarn-mode)
|
||||
* Contribute
|
||||
* [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html)
|
||||
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Apache Zeppelin on Spark cluster mode"
|
||||
description: ""
|
||||
description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using docker scripts."
|
||||
group: install
|
||||
---
|
||||
<!--
|
||||
|
|
@ -56,12 +56,12 @@ spark_standalone bash;
|
|||
```
|
||||
|
||||
### 3. Configure Spark interpreter in Zeppelin
|
||||
Set Spark master as `spark://localhost:7077` in Zeppelin **Interpreters** setting page.
|
||||
Set Spark master as `spark://<hostname>:7077` in Zeppelin **Interpreters** setting page.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/standalone_conf.png" />
|
||||
|
||||
### 4. Run Zeppelin with Spark interpreter
|
||||
After running single paragraph with Spark interpreter in Zeppelin, browse `https://localhost:8080` and check whether Spark cluster is running well or not.
|
||||
After running single paragraph with Spark interpreter in Zeppelin, browse `https://<hostname>:8080` and check whether Spark cluster is running well or not.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/spark_ui.png" />
|
||||
|
||||
|
|
@ -72,3 +72,71 @@ ps -ef | grep spark
|
|||
```
|
||||
|
||||
|
||||
## Spark on YARN mode
|
||||
You can simply set up [Spark on YARN](http://spark.apache.org/docs/latest/running-on-yarn.html) docker environment with below steps.
|
||||
|
||||
> **Note :** Since Apache Zeppelin and Spark use same `8080` port for their web UI, you might need to change `zeppelin.server.port` in `conf/zeppelin-site.xml`.
|
||||
|
||||
### 1. Build Docker file
|
||||
You can find docker script files under `scripts/docker/spark-cluster-managers`.
|
||||
|
||||
```
|
||||
cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_yarn
|
||||
docker build -t "spark_yarn" .
|
||||
```
|
||||
|
||||
### 2. Run docker
|
||||
|
||||
```
|
||||
docker run -it \
|
||||
-p 5000:5000 \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-p 8088:8088 \
|
||||
-p 8042:8042 \
|
||||
-p 8030:8030 \
|
||||
-p 8031:8031 \
|
||||
-p 8032:8032 \
|
||||
-p 8033:8033 \
|
||||
-p 8080:8080 \
|
||||
-p 7077:7077 \
|
||||
-p 8888:8888 \
|
||||
-p 8081:8081 \
|
||||
-p 50010:50010 \
|
||||
-p 50075:50075 \
|
||||
-p 50020:50020 \
|
||||
-p 50070:50070 \
|
||||
--name spark_yarn \
|
||||
-h sparkmaster \
|
||||
spark_yarn bash;
|
||||
```
|
||||
|
||||
### 3. Verify running Spark on YARN.
|
||||
|
||||
You can simply verify the processes of Spark and YARN is running well in Docker with below command.
|
||||
|
||||
```
|
||||
ps -ef
|
||||
```
|
||||
|
||||
You can also check each application web UI for HDFS on `http://<hostname>:50070/`, YARN on `http://<hostname>:8088/cluster` and Spark on `http://<hostname>:8080/`.
|
||||
|
||||
### 4. Configure Spark interpreter in Zeppelin
|
||||
Set following configurations to `conf/zeppelin-env.sh`.
|
||||
|
||||
```
|
||||
export MASTER=yarn-client
|
||||
export HADOOP_CONF_DIR=[your_hadoop_conf_path]
|
||||
export SPARK_HOME=[your_spark_home_path]
|
||||
```
|
||||
|
||||
`HADOOP_CONF_DIR`(Hadoop configuration path) is defined in `/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf`.
|
||||
|
||||
Don't forget to set Spark `master` as `yarn-client` in Zeppelin **Interpreters** setting page like below.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png" />
|
||||
|
||||
### 5. Run Zeppelin with Spark interpreter
|
||||
After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/yarn_applications.png" />
|
||||
|
|
|
|||
|
|
@ -198,7 +198,10 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
<td>
|
||||
400 if the input json is empty <br/>
|
||||
500 for any other errors
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Sample JSON input</td>
|
||||
|
|
@ -219,7 +222,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -249,7 +254,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -298,7 +305,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -328,7 +337,9 @@ The role of registered interpreters, settings and interpreters group are describ
|
|||
"dependencies": [
|
||||
{
|
||||
"groupArtifactVersion": "groupId:artifactId:version",
|
||||
"exclusions": "groupId:artifactId"
|
||||
"exclusions": [
|
||||
"groupId:artifactId"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -298,7 +298,10 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```POST``` method runs all paragraphs in the given notebook id.
|
||||
<td>
|
||||
This ```POST``` method runs all paragraphs in the given notebook id. <br />
|
||||
If you can not find Notebook id 404 returns.
|
||||
If there is a problem with the interpreter returns a 412 error.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -311,12 +314,29 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
<td> 404 or 412</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response </td>
|
||||
<td><pre>{"status": "OK"}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON error response </td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
"status": "NOT_FOUND",
|
||||
"message": "note not found."
|
||||
}
|
||||
</pre><br />
|
||||
<pre>
|
||||
{
|
||||
"status": "PRECONDITION_FAILED",
|
||||
"message": "paragraph_1469771130099_-278315611 Not selected or Invalid Interpreter bind"
|
||||
}
|
||||
</pre>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
|
|
@ -392,6 +412,43 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Get the status of a single paragraph
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method gets the status of a single paragraph by the given notebook and paragraph id.
|
||||
The body field of the returned JSON contains of the array that compose of the paragraph id, paragraph status, paragraph finish date, paragraph started date.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/job/[notebookId]/[paragraphId]```</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Success code</td>
|
||||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response </td>
|
||||
<td><pre>
|
||||
{
|
||||
"status": "OK",
|
||||
"body": {
|
||||
"id":"20151121-212654\_766735423",
|
||||
"status":"FINISHED",
|
||||
"finished":"Tue Nov 24 14:21:40 KST 2015",
|
||||
"started":"Tue Nov 24 14:21:39 KST 2015"
|
||||
}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Run a paragraph
|
||||
<table class="table-configuration">
|
||||
|
|
|
|||
|
|
@ -112,10 +112,36 @@ To learn more about Apache Shiro Realm, please check [this documentation](http:/
|
|||
We also provide community custom Realms.
|
||||
|
||||
### Active Directory
|
||||
TBD
|
||||
|
||||
```
|
||||
activeDirectoryRealm = org.apache.zeppelin.server.ActiveDirectoryGroupRealm
|
||||
activeDirectoryRealm.systemUsername = userNameA
|
||||
activeDirectoryRealm.systemPassword = passwordA
|
||||
activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks
|
||||
activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
||||
activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
|
||||
activeDirectoryRealm.authorizationCachingEnabled = false
|
||||
```
|
||||
|
||||
|
||||
Also instead of specifying systemPassword in clear text in shiro.ini administrator can choose to specify the same in "hadoop credential".
|
||||
Create a keystore file using the hadoop credential commandline, for this the hadoop commons should be in the classpath
|
||||
`hadoop credential create activeDirectoryRealm.systempassword -provider jceks://file/user/zeppelin/conf/zeppelin.jceks`
|
||||
|
||||
Change the following values in the Shiro.ini file, and uncomment the line:
|
||||
`activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/conf/zeppelin.jceks`
|
||||
|
||||
### LDAP
|
||||
TBD
|
||||
|
||||
```
|
||||
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
|
||||
# search base for ldap groups (only relevant for LdapGroupRealm):
|
||||
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
|
||||
ldapRealm.contextFactory.url = ldap://ldap.test.com:389
|
||||
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
|
||||
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
|
||||
```
|
||||
|
||||
### ZeppelinHub
|
||||
[ZeppelinHub](https://www.zeppelinhub.com) is a service that synchronize your Apache Zeppelin notebooks and enables you to collaborate easily.
|
||||
|
|
|
|||
|
|
@ -54,23 +54,15 @@ import java.util.Properties;
|
|||
* zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
|
||||
*/
|
||||
public class HbaseInterpreter extends Interpreter {
|
||||
public static final String HBASE_HOME = "hbase.home";
|
||||
public static final String HBASE_RUBY_SRC = "hbase.ruby.sources";
|
||||
public static final String HBASE_TEST_MODE = "zeppelin.hbase.test.mode";
|
||||
|
||||
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
|
||||
private ScriptingContainer scriptingContainer;
|
||||
|
||||
private StringWriter writer;
|
||||
|
||||
static {
|
||||
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
|
||||
new InterpreterPropertyBuilder()
|
||||
.add("hbase.home",
|
||||
getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
|
||||
"Installation directory of HBase")
|
||||
.add("hbase.ruby.sources", "lib/ruby",
|
||||
"Path to Ruby scripts relative to 'hbase.home'")
|
||||
.add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
|
||||
.build());
|
||||
}
|
||||
|
||||
public HbaseInterpreter(Properties property) {
|
||||
super(property);
|
||||
}
|
||||
|
|
@ -81,9 +73,9 @@ public class HbaseInterpreter extends Interpreter {
|
|||
this.writer = new StringWriter();
|
||||
scriptingContainer.setOutput(this.writer);
|
||||
|
||||
if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
|
||||
String hbase_home = getProperty("hbase.home");
|
||||
String ruby_src = getProperty("hbase.ruby.sources");
|
||||
if (!Boolean.parseBoolean(getProperty(HBASE_TEST_MODE))) {
|
||||
String hbase_home = getProperty(HBASE_HOME);
|
||||
String ruby_src = getProperty(HBASE_RUBY_SRC);
|
||||
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
|
||||
|
||||
logger.info("Home:" + hbase_home);
|
||||
|
|
@ -98,7 +90,7 @@ public class HbaseInterpreter extends Interpreter {
|
|||
logger.info("Absolute Ruby Source:" + abs_ruby_src.toString());
|
||||
// hirb.rb:41 requires the following system property to be set.
|
||||
Properties sysProps = System.getProperties();
|
||||
sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString());
|
||||
sysProps.setProperty(HBASE_RUBY_SRC, abs_ruby_src.toString());
|
||||
|
||||
Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb");
|
||||
try {
|
||||
|
|
|
|||
25
hbase/src/main/resources/interpreter-setting.json
Normal file
25
hbase/src/main/resources/interpreter-setting.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[
|
||||
{
|
||||
"group": "hbase",
|
||||
"name": "hbase",
|
||||
"className": "org.apache.zeppelin.hbase.HbaseInterpreter",
|
||||
"properties": {
|
||||
"hbase.home": {
|
||||
"envName": "HBASE_HOME",
|
||||
"propertyName": "hbase.home",
|
||||
"defaultValue": "/usr/lib/hbase/",
|
||||
"description": "Installation directory of HBase"
|
||||
},
|
||||
"hbase.ruby.sources": {
|
||||
"propertyName": "hbase.ruby.sources",
|
||||
"defaultValue": "lib/ruby",
|
||||
"description": "Path to Ruby scripts relative to 'hbase.home'"
|
||||
},
|
||||
"zeppelin.hbase.test.mode": {
|
||||
"propertyName": "zeppelin.hbase.test.mode",
|
||||
"defaultValue": "false",
|
||||
"description": "Disable checks for unit and manual tests"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
1
pom.xml
1
pom.xml
|
|
@ -486,6 +486,7 @@
|
|||
<exclude>docs/sitemap.txt</exclude>
|
||||
<exclude>docs/search_data.json</exclude>
|
||||
<exclude>**/dependency-reduced-pom.xml</exclude>
|
||||
<exclude>docs/CONTRIBUTING.md</exclude>
|
||||
|
||||
<!-- bundled from anchor -->
|
||||
<exclude>docs/assets/themes/zeppelin/js/anchor.min.js</exclude>
|
||||
|
|
|
|||
|
|
@ -20,11 +20,11 @@
|
|||
import sys
|
||||
import signal
|
||||
import base64
|
||||
|
||||
from io import BytesIO
|
||||
try:
|
||||
import StringIO as io
|
||||
from StringIO import StringIO
|
||||
except ImportError:
|
||||
import io as io
|
||||
from io import StringIO
|
||||
|
||||
def intHandler(signum, frame): # Set the signal handler
|
||||
print ("Paragraph interrupted")
|
||||
|
|
@ -141,14 +141,14 @@ class PyZeppelinContext(object):
|
|||
"""Pretty prints DF using Table Display System
|
||||
"""
|
||||
limit = len(df) > self.max_result
|
||||
header_buf = io.StringIO("")
|
||||
header_buf = StringIO("")
|
||||
header_buf.write(str(df.columns[0]))
|
||||
for col in df.columns[1:]:
|
||||
header_buf.write("\t")
|
||||
header_buf.write(str(col))
|
||||
header_buf.write("\n")
|
||||
|
||||
body_buf = io.StringIO("")
|
||||
body_buf = StringIO("")
|
||||
rows = df.head(self.max_result).values if limit else df.values
|
||||
for row in rows:
|
||||
body_buf.write(str(row[0]))
|
||||
|
|
@ -164,23 +164,27 @@ class PyZeppelinContext(object):
|
|||
#)
|
||||
body_buf.close(); header_buf.close()
|
||||
|
||||
def show_matplotlib(self, p, width="100%", height="100%",
|
||||
fmt='png', **kwargs):
|
||||
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
|
||||
**kwargs):
|
||||
"""Matplotlib show function
|
||||
"""
|
||||
img = io.StringIO()
|
||||
if fmt == 'png':
|
||||
if fmt == "png":
|
||||
img = BytesIO()
|
||||
p.savefig(img, format=fmt)
|
||||
html = "%html <img src={img} width={width}, height={height}>"
|
||||
img_str = "data:image/png;base64,"
|
||||
img_str = b"data:image/png;base64,"
|
||||
img_str += base64.b64encode(img.getvalue().strip())
|
||||
elif fmt == 'svg':
|
||||
img_tag = "<img src={img} style='width={width};height:{height}'>"
|
||||
# Decoding is necessary for Python 3 compability
|
||||
img_str = img_str.decode("ascii")
|
||||
img_str = img_tag.format(img=img_str, width=width, height=height)
|
||||
elif fmt == "svg":
|
||||
img = StringIO()
|
||||
p.savefig(img, format=fmt)
|
||||
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
|
||||
img_str = img.getvalue()
|
||||
else:
|
||||
raise ValueError("fmt must be 'png' or 'svg'")
|
||||
|
||||
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
|
||||
print(html.format(width=width, height=height, img=img_str))
|
||||
img.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.zeppelin.rinterpreter
|
||||
|
||||
import java.io.{BufferedInputStream, File, FileInputStream}
|
||||
import java.nio.file.{Files, Paths}
|
||||
import java.util._
|
||||
|
||||
|
|
@ -110,10 +111,10 @@ object RInterpreter {
|
|||
|
||||
// These are the additional properties we need on top of the ones provided by the spark interpreters
|
||||
lazy val props: Map[String, InterpreterProperty] = new InterpreterPropertyBuilder()
|
||||
.add("rhadoop.cmd", SparkInterpreter.getSystemDefault("rhadoop.cmd", "HADOOP_CMD", ""), "Usually /usr/bin/hadoop")
|
||||
.add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("rhadoop.cmd", "HADOOP_STREAMING", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
|
||||
.add("rscala.debug", SparkInterpreter.getSystemDefault("rscala.debug","RSCALA_DEBUG", "false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested
|
||||
.add("rscala.timeout", SparkInterpreter.getSystemDefault("rscala.timeout","RSCALA_TIMEOUT", "60"), "Timeout for rScala") // TEST: Implemented but not tested
|
||||
.add("rhadoop.cmd", SparkInterpreter.getSystemDefault("HADOOP_CMD", "rhadoop.cmd", ""), "Usually /usr/bin/hadoop")
|
||||
.add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("HADOOP_STREAMING", "rhadooop.streamingjar", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar")
|
||||
.add("rscala.debug", SparkInterpreter.getSystemDefault("RSCALA_DEBUG", "rscala.debug","false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested
|
||||
.add("rscala.timeout", SparkInterpreter.getSystemDefault("RSCALA_TIMEOUT", "rscala.timeout","60"), "Timeout for rScala") // TEST: Implemented but not tested
|
||||
.build
|
||||
|
||||
def getProps() = {
|
||||
|
|
@ -141,8 +142,15 @@ object RInterpreter {
|
|||
}
|
||||
|
||||
def dataURI(file : String, mime : String) : String = {
|
||||
val data: String = Source.fromFile(file).getLines().mkString("\n")
|
||||
s"""data:${mime};base64,""" + StringUtils.newStringUtf8(Base64.encodeBase64(data.getBytes(), false))
|
||||
val fp = new File(file)
|
||||
val fdata = new Array[Byte](fp.length().toInt)
|
||||
val fin = new BufferedInputStream(new FileInputStream(fp))
|
||||
try {
|
||||
fin.read(fdata)
|
||||
} finally {
|
||||
fin.close()
|
||||
}
|
||||
s"""data:${mime};base64,""" + StringUtils.newStringUtf8(Base64.encodeBase64(fdata, false))
|
||||
}
|
||||
|
||||
// The purpose here is to deal with knitr producing HTML with script and css tags outside the <body>
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
FROM centos:centos6
|
||||
MAINTAINER hsshim@nflabs.com
|
||||
|
||||
ENV SPARK_PROFILE 1.6
|
||||
ENV SPARK_VERSION 1.6.2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
FROM centos:centos6
|
||||
|
||||
ENV SPARK_PROFILE 2.0
|
||||
ENV SPARK_VERSION 2.0.0
|
||||
ENV HADOOP_PROFILE 2.7
|
||||
ENV HADOOP_VERSION 2.7.0
|
||||
|
||||
# Update the image with the latest packages
|
||||
RUN yum update -y; yum clean all
|
||||
|
||||
# Get utils
|
||||
RUN yum install -y \
|
||||
wget \
|
||||
tar \
|
||||
curl \
|
||||
&& \
|
||||
yum clean all
|
||||
|
||||
# Remove old jdk
|
||||
RUN yum remove java; yum remove jdk
|
||||
|
||||
# install jdk7
|
||||
RUN yum install -y java-1.7.0-openjdk-devel
|
||||
ENV JAVA_HOME /usr/lib/jvm/java
|
||||
ENV PATH $PATH:$JAVA_HOME/bin
|
||||
|
||||
# install hadoop
|
||||
RUN yum install -y curl which tar sudo openssh-server openssh-clients rsync
|
||||
|
||||
# hadoop
|
||||
RUN curl -s https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar -xz -C /usr/local/
|
||||
RUN cd /usr/local && ln -s ./hadoop-$HADOOP_VERSION hadoop
|
||||
|
||||
ENV HADOOP_PREFIX /usr/local/hadoop
|
||||
ENV HADOOP_COMMON_HOME /usr/local/hadoop
|
||||
ENV HADOOP_HDFS_HOME /usr/local/hadoop
|
||||
ENV HADOOP_MAPRED_HOME /usr/local/hadoop
|
||||
ENV HADOOP_YARN_HOME /usr/local/hadoop
|
||||
ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
|
||||
|
||||
RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/jre-1.7.0-openjdk.x86_64\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
|
||||
RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
|
||||
|
||||
RUN mkdir $HADOOP_PREFIX/input
|
||||
RUN cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
|
||||
|
||||
# hadoop configurations
|
||||
ADD hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml
|
||||
ADD hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
|
||||
ADD hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
|
||||
ADD hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
|
||||
|
||||
RUN mkdir /data/
|
||||
RUN chmod 777 /data/
|
||||
RUN $HADOOP_PREFIX/bin/hdfs namenode -format
|
||||
|
||||
RUN rm /usr/local/hadoop/lib/native/*
|
||||
RUN curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-$HADOOP_VERSION.tar|tar -x -C /usr/local/hadoop/lib/native/
|
||||
|
||||
# install spark
|
||||
RUN curl -s http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
|
||||
RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
|
||||
ENV SPARK_HOME /usr/local/spark
|
||||
|
||||
ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
|
||||
ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin
|
||||
|
||||
# passwordless ssh
|
||||
RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
|
||||
RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
|
||||
RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa
|
||||
RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
|
||||
|
||||
ADD ssh_config /root/.ssh/config
|
||||
RUN chmod 600 /root/.ssh/config
|
||||
RUN chown root:root /root/.ssh/config
|
||||
RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh
|
||||
|
||||
# update boot script
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chown root.root /etc/entrypoint.sh
|
||||
RUN chmod 700 /etc/entrypoint.sh
|
||||
|
||||
# Hdfs ports
|
||||
EXPOSE 50010 50020 50070 50075 50090
|
||||
# Mapred ports
|
||||
EXPOSE 9000 9001
|
||||
#Yarn ports
|
||||
EXPOSE 8030 8031 8032 8033 8040 8042 8088
|
||||
#spark
|
||||
EXPOSE 8080 7077 8888 8081
|
||||
|
||||
ENTRYPOINT ["/etc/entrypoint.sh"]
|
||||
60
scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh
Executable file
60
scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh
Executable file
|
|
@ -0,0 +1,60 @@
|
|||
#!/bin/bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
echo 'hadoop' |passwd root --stdin
|
||||
|
||||
: ${HADOOP_PREFIX:=/usr/local/hadoop}
|
||||
|
||||
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
|
||||
|
||||
rm /tmp/*.pid
|
||||
|
||||
# installing libraries if any - (resource urls added comma separated to the ACP system variable)
|
||||
cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
|
||||
|
||||
cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
|
||||
|
||||
# start hadoop
|
||||
service sshd start
|
||||
$HADOOP_PREFIX/sbin/start-dfs.sh
|
||||
$HADOOP_PREFIX/sbin/start-yarn.sh
|
||||
|
||||
$HADOOP_PREFIX/bin/hdfs dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE/lib /spark
|
||||
|
||||
# start spark
|
||||
export SPARK_MASTER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
|
||||
-Dspark.broadcast.port=7003 -Dspark.replClassServer.port=7004
|
||||
-Dspark.blockManager.port=7005 -Dspark.executor.port=7006
|
||||
-Dspark.ui.port=4040 -Dspark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory"
|
||||
export SPARK_WORKER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
|
||||
-Dspark.broadcast.port=7003 -Dspark.replClassServer.port=7004
|
||||
-Dspark.blockManager.port=7005 -Dspark.executor.port=7006
|
||||
-Dspark.ui.port=4040 -Dspark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory"
|
||||
|
||||
export SPARK_MASTER_PORT=7077
|
||||
|
||||
cd /usr/local/spark/sbin
|
||||
./start-master.sh
|
||||
./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
|
||||
|
||||
CMD=${1:-"exit 0"}
|
||||
if [[ "$CMD" == "-d" ]];
|
||||
then
|
||||
service sshd stop
|
||||
/usr/sbin/sshd -D -d
|
||||
else
|
||||
/bin/bash -c "$*"
|
||||
fi
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<configuration>
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://0.0.0.0:9000</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<configuration>
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.data.dir</name>
|
||||
<value>/data/hdfs</value>
|
||||
<final>true</final>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.permissions</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.client.use.datanode.hostname</name>
|
||||
<value>true</value>
|
||||
<description>Whether clients should use datanode hostnames when
|
||||
connecting to datanodes.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.use.datanode.hostname</name>
|
||||
<value>true</value>
|
||||
<description>Whether datanodes should use datanode hostnames when
|
||||
connecting to other datanodes for data transfer.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.address</name>
|
||||
<value>0.0.0.0:50010</value>
|
||||
<description>
|
||||
The address where the datanode server will listen to.
|
||||
If the port is 0 then the server will start on a free port.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.http.address</name>
|
||||
<value>0.0.0.0:50075</value>
|
||||
<description>
|
||||
The datanode http server address and port.
|
||||
If the port is 0 then the server will start on a free port.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.ipc.address</name>
|
||||
<value>0.0.0.0:50020</value>
|
||||
<description>
|
||||
The datanode ipc server address and port.
|
||||
If the port is 0 then the server will start on a free port.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<configuration>
|
||||
<property>
|
||||
<name>mapreduce.framework.name</name>
|
||||
<value>yarn</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<configuration>
|
||||
<property>
|
||||
<name>yarn.resourcemanager.scheduler.address</name>
|
||||
<value>0.0.0.0:8030</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.resourcemanager.address</name>
|
||||
<value>0.0.0.0:8032</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.resourcemanager.webapp.address</name>
|
||||
<value>0.0.0.0:8088</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.resourcemanager.resource-tracker.address</name>
|
||||
<value>0.0.0.0:8031</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.resourcemanager.admin.address</name>
|
||||
<value>0.0.0.0:8033</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.application.classpath</name>
|
||||
<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*, /usr/local/hadoop/share/spark/*</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
Host *
|
||||
UserKnownHostsFile /dev/null
|
||||
StrictHostKeyChecking no
|
||||
LogLevel quiet
|
||||
|
|
@ -75,6 +75,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
private ByteArrayOutputStream input;
|
||||
private String scriptPath;
|
||||
boolean pythonscriptRunning = false;
|
||||
private static final int MAX_TIMEOUT_SEC = 10;
|
||||
|
||||
public PySparkInterpreter(Properties property) {
|
||||
super(property);
|
||||
|
|
@ -316,7 +317,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
long startTime = System.currentTimeMillis();
|
||||
while (pythonScriptInitialized == false
|
||||
&& pythonscriptRunning
|
||||
&& System.currentTimeMillis() - startTime < 10 * 1000) {
|
||||
&& System.currentTimeMillis() - startTime < MAX_TIMEOUT_SEC * 1000) {
|
||||
try {
|
||||
pythonScriptInitializeNotifier.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
|
|
@ -423,8 +424,15 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
}
|
||||
|
||||
synchronized (statementFinishedNotifier) {
|
||||
while (statementOutput == null) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
while (statementOutput == null
|
||||
&& pythonScriptInitialized == false
|
||||
&& pythonscriptRunning) {
|
||||
try {
|
||||
if (System.currentTimeMillis() - startTime < MAX_TIMEOUT_SEC * 1000) {
|
||||
logger.error("pyspark completion didn't have response for {}sec.", MAX_TIMEOUT_SEC);
|
||||
break;
|
||||
}
|
||||
statementFinishedNotifier.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
// not working
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.SparkContext;
|
||||
|
|
@ -300,7 +301,9 @@ public class SparkInterpreter extends Interpreter {
|
|||
String execUri = System.getenv("SPARK_EXECUTOR_URI");
|
||||
conf.setAppName(getProperty("spark.app.name"));
|
||||
|
||||
conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath());
|
||||
if (outputDir != null) {
|
||||
conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath());
|
||||
}
|
||||
|
||||
if (execUri != null) {
|
||||
conf.set("spark.executor.uri", execUri);
|
||||
|
|
@ -593,7 +596,11 @@ public class SparkInterpreter extends Interpreter {
|
|||
argList.add("-Yrepl-class-based");
|
||||
argList.add("-Yrepl-outdir");
|
||||
argList.add(outputDir.getAbsolutePath());
|
||||
|
||||
if (conf.contains("spark.jars")) {
|
||||
String jars = StringUtils.join(conf.get("spark.jars").split(","), File.separator);
|
||||
argList.add("-classpath");
|
||||
argList.add(jars);
|
||||
}
|
||||
|
||||
scala.collection.immutable.List<String> list =
|
||||
JavaConversions.asScalaBuffer(argList).toList();
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ class Utils {
|
|||
return true;
|
||||
} catch (ClassNotFoundException e) {
|
||||
return false;
|
||||
} catch (IncompatibleClassChangeError e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ The following components are provided under Apache License.
|
|||
(Apache 2.0) Utility classes for Jetty (org.mortbay.jetty:jetty-util:6.1.26 - http://javadox.com/org.mortbay.jetty/jetty/6.1.26/overview-tree.html)
|
||||
(Apache 2.0) Servlet API (org.mortbay.jetty:servlet-api:2.5-20081211 - https://en.wikipedia.org/wiki/Jetty_(web_server))
|
||||
(Apache 2.0) Google HTTP Client Library for Java (com.google.http-client:google-http-client-jackson2:1.21.0 - https://github.com/google/google-http-java-client/tree/dev/google-http-client-jackson2)
|
||||
(Apache 2.0) angular-esri-map (https://github.com/Esri/angular-esri-map)
|
||||
|
||||
========================================================================
|
||||
MIT licenses
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@
|
|||
<properties>
|
||||
<cxf.version>2.7.7</cxf.version>
|
||||
<commons.httpclient.version>4.3.6</commons.httpclient.version>
|
||||
<hadoop-common.version>2.6.0</hadoop-common.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
|
|
@ -205,6 +206,61 @@
|
|||
<artifactId>commons-collections</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>${hadoop-common.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-json</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-server</artifactId>
|
||||
</exclusion>
|
||||
|
||||
|
||||
<exclusion>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.jackrabbit</groupId>
|
||||
<artifactId>jackrabbit-webdav</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-httpclient</groupId>
|
||||
<artifactId>commons-httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.eclipse.jgit</groupId>
|
||||
<artifactId>org.eclipse.jgit</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.jcraft</groupId>
|
||||
<artifactId>jsch</artifactId>
|
||||
</exclusion>
|
||||
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.quartz-scheduler</groupId>
|
||||
<artifactId>quartz</artifactId>
|
||||
|
|
|
|||
|
|
@ -90,6 +90,9 @@ public class InterpreterRestApi {
|
|||
try {
|
||||
NewInterpreterSettingRequest request =
|
||||
gson.fromJson(message, NewInterpreterSettingRequest.class);
|
||||
if (request == null) {
|
||||
return new JsonResponse<>(Status.BAD_REQUEST).build();
|
||||
}
|
||||
Properties p = new Properties();
|
||||
p.putAll(request.getProperties());
|
||||
InterpreterSetting interpreterSetting = interpreterFactory
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ import com.google.common.collect.Sets;
|
|||
import com.google.common.reflect.TypeToken;
|
||||
import com.google.gson.Gson;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.scheduler.Job;
|
||||
import org.apache.zeppelin.utils.InterpreterBindingUtils;
|
||||
import org.quartz.CronExpression;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -116,8 +118,8 @@ public class NotebookRestApi {
|
|||
* TODO(jl): Fixed the type of HashSet
|
||||
* https://issues.apache.org/jira/browse/ZEPPELIN-1162
|
||||
*/
|
||||
HashMap<String, HashSet> permMap =
|
||||
gson.fromJson(req, new TypeToken<HashMap<String, HashSet>>() {
|
||||
HashMap<String, HashSet<String>> permMap =
|
||||
gson.fromJson(req, new TypeToken<HashMap<String, HashSet<String>>>() {
|
||||
}.getType());
|
||||
Note note = notebook.getNote(noteId);
|
||||
String principal = SecurityUtils.getPrincipal();
|
||||
|
|
@ -133,9 +135,9 @@ public class NotebookRestApi {
|
|||
ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))).build();
|
||||
}
|
||||
|
||||
HashSet readers = permMap.get("readers");
|
||||
HashSet owners = permMap.get("owners");
|
||||
HashSet writers = permMap.get("writers");
|
||||
HashSet<String> readers = permMap.get("readers");
|
||||
HashSet<String> owners = permMap.get("owners");
|
||||
HashSet<String> writers = permMap.get("writers");
|
||||
// Set readers, if writers and owners is empty -> set to user requesting the change
|
||||
if (readers != null && !readers.isEmpty()) {
|
||||
if (writers.isEmpty()) {
|
||||
|
|
@ -477,7 +479,14 @@ public class NotebookRestApi {
|
|||
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
|
||||
}
|
||||
|
||||
note.runAll();
|
||||
try {
|
||||
note.runAll();
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Exception from run", ex);
|
||||
return new JsonResponse<>(Status.PRECONDITION_FAILED,
|
||||
ex.getMessage() + "- Not selected or Invalid Interpreter bind").build();
|
||||
}
|
||||
|
||||
return new JsonResponse<>(Status.OK).build();
|
||||
}
|
||||
|
||||
|
|
@ -528,6 +537,35 @@ public class NotebookRestApi {
|
|||
return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get notebook paragraph job status REST API
|
||||
*
|
||||
* @param notebookId ID of Notebook
|
||||
* @param paragraphId ID of Paragraph
|
||||
* @return JSON with status.OK
|
||||
* @throws IOException, IllegalArgumentException
|
||||
*/
|
||||
@GET
|
||||
@Path("job/{notebookId}/{paragraphId}")
|
||||
@ZeppelinApi
|
||||
public Response getNoteParagraphJobStatus(@PathParam("notebookId") String notebookId,
|
||||
@PathParam("paragraphId") String paragraphId)
|
||||
throws IOException, IllegalArgumentException {
|
||||
LOG.info("get notebook paragraph job status.");
|
||||
Note note = notebook.getNote(notebookId);
|
||||
if (note == null) {
|
||||
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
|
||||
}
|
||||
|
||||
Paragraph paragraph = note.getParagraph(paragraphId);
|
||||
if (paragraph == null) {
|
||||
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
|
||||
}
|
||||
|
||||
return new JsonResponse<>(Status.OK, null, note.generateSingleParagraphInfo(paragraphId)).
|
||||
build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph job REST API
|
||||
*
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@
|
|||
*/
|
||||
package org.apache.zeppelin.server;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.alias.CredentialProvider;
|
||||
import org.apache.hadoop.security.alias.CredentialProviderFactory;
|
||||
import org.apache.shiro.authc.AuthenticationInfo;
|
||||
import org.apache.shiro.authc.AuthenticationToken;
|
||||
import org.apache.shiro.authc.SimpleAuthenticationInfo;
|
||||
|
|
@ -55,6 +59,13 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
|
|||
|
||||
private static final String ROLE_NAMES_DELIMETER = ",";
|
||||
|
||||
String KEYSTORE_PASS = "activeDirectoryRealm.systemPassword";
|
||||
private String hadoopSecurityCredentialPath;
|
||||
|
||||
public void setHadoopSecurityCredentialPath(String hadoopSecurityCredentialPath) {
|
||||
this.hadoopSecurityCredentialPath = hadoopSecurityCredentialPath;
|
||||
}
|
||||
|
||||
/*--------------------------------------------
|
||||
| I N S T A N C E V A R I A B L E S |
|
||||
============================================*/
|
||||
|
|
@ -91,13 +102,36 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
|
|||
defaultFactory.setSearchBase(this.searchBase);
|
||||
defaultFactory.setUrl(this.url);
|
||||
defaultFactory.setSystemUsername(this.systemUsername);
|
||||
defaultFactory.setSystemPassword(this.systemPassword);
|
||||
defaultFactory.setSystemPassword(getSystemPassword());
|
||||
this.ldapContextFactory = defaultFactory;
|
||||
}
|
||||
|
||||
return this.ldapContextFactory;
|
||||
}
|
||||
|
||||
private String getSystemPassword() {
|
||||
String password = "";
|
||||
if (StringUtils.isEmpty(this.hadoopSecurityCredentialPath)) {
|
||||
password = this.systemPassword;
|
||||
} else {
|
||||
try {
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
||||
this.hadoopSecurityCredentialPath);
|
||||
CredentialProvider provider =
|
||||
CredentialProviderFactory.getProviders(configuration).get(0);
|
||||
CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(
|
||||
KEYSTORE_PASS);
|
||||
if (credEntry != null) {
|
||||
password = new String(credEntry.getCredential());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
}
|
||||
return password;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds an {@link AuthenticationInfo} object by querying the active directory LDAP context for
|
||||
* the specified username. This method binds to the LDAP server using the provided username
|
||||
|
|
@ -293,3 +327,4 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -119,6 +119,15 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
|
|||
delete.releaseConnection();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSettingsCreateWithEmptyJson() throws IOException {
|
||||
// Call Create Setting REST API
|
||||
PostMethod post = httpPost("/interpreter/setting/", "");
|
||||
LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
|
||||
assertThat("test create method:", post, isBadRequest());
|
||||
post.releaseConnection();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInterpreterAutoBinding() throws IOException {
|
||||
// create note
|
||||
|
|
|
|||
|
|
@ -122,6 +122,28 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
|
|||
ZeppelinServer.notebook.removeNote(note2.getId(), null);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetNoteParagraphJobStatus() throws IOException {
|
||||
Note note1 = ZeppelinServer.notebook.createNote(null);
|
||||
note1.addParagraph();
|
||||
|
||||
String paragraphId = note1.getLastParagraph().getId();
|
||||
|
||||
GetMethod get = httpGet("/notebook/job/" + note1.getId() + "/" + paragraphId);
|
||||
assertThat(get, isAllowed());
|
||||
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
|
||||
}.getType());
|
||||
Map<String, Set<String>> paragraphStatus = (Map<String, Set<String>>) resp.get("body");
|
||||
|
||||
// Check id and status have proper value
|
||||
assertEquals(paragraphStatus.get("id"), paragraphId);
|
||||
assertEquals(paragraphStatus.get("status"), "READY");
|
||||
|
||||
//cleanup
|
||||
ZeppelinServer.notebook.removeNote(note1.getId(), null);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@
|
|||
"bootstrap3-dialog": "bootstrap-dialog#~1.34.7",
|
||||
"handsontable": "~0.24.2",
|
||||
"moment-duration-format": "^1.3.0",
|
||||
"select2": "^4.0.3"
|
||||
"select2": "^4.0.3",
|
||||
"angular-esri-map": "~2.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"angular-mocks": "1.5.0"
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@
|
|||
'xeditable',
|
||||
'ngToast',
|
||||
'focus-if',
|
||||
'ngResource'
|
||||
'ngResource',
|
||||
'esri.map'
|
||||
])
|
||||
.filter('breakFilter', function() {
|
||||
return function(text) {
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro
|
|||
var initNotebook = function() {
|
||||
websocketMsgSrv.getNotebook($routeParams.noteId);
|
||||
websocketMsgSrv.listRevisionHistory($routeParams.noteId);
|
||||
|
||||
var currentRoute = $route.current;
|
||||
if (currentRoute) {
|
||||
setTimeout(
|
||||
|
|
@ -333,6 +332,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro
|
|||
} else {
|
||||
$scope.viewOnly = $scope.note.config.looknfeel === 'report' ? true : false;
|
||||
}
|
||||
$scope.note.paragraphs[0].focus = true;
|
||||
$rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -20,32 +20,44 @@ limitations under the License.
|
|||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('table')}"
|
||||
ng-click="setGraphMode('table', true)" ><i class="fa fa-table"></i>
|
||||
ng-click="setGraphMode('table', true)"
|
||||
tooltip="Table" tooltip-placement="bottom"><i class="fa fa-table"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('multiBarChart')}"
|
||||
ng-click="setGraphMode('multiBarChart', true)"><i class="fa fa-bar-chart"></i>
|
||||
ng-click="setGraphMode('multiBarChart', true)"
|
||||
tooltip="Bar Chart" tooltip-placement="bottom"><i class="fa fa-bar-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('pieChart')}"
|
||||
ng-click="setGraphMode('pieChart', true)"><i class="fa fa-pie-chart"></i>
|
||||
ng-click="setGraphMode('pieChart', true)"
|
||||
tooltip="Pie Chart" tooltip-placement="bottom"><i class="fa fa-pie-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('stackedAreaChart')}"
|
||||
ng-click="setGraphMode('stackedAreaChart', true)"><i class="fa fa-area-chart"></i>
|
||||
ng-click="setGraphMode('stackedAreaChart', true)"
|
||||
tooltip="Area Chart" tooltip-placement="bottom"><i class="fa fa-area-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('lineChart') || isGraphMode('lineWithFocusChart')}"
|
||||
ng-click="paragraph.config.graph.lineWithFocus ? setGraphMode('lineWithFocusChart', true) : setGraphMode('lineChart', true)"><i class="fa fa-line-chart"></i>
|
||||
ng-click="paragraph.config.graph.lineWithFocus ? setGraphMode('lineWithFocusChart', true) : setGraphMode('lineChart', true)"
|
||||
tooltip="Line Chart" tooltip-placement="bottom"><i class="fa fa-line-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('scatterChart')}"
|
||||
ng-click="setGraphMode('scatterChart', true)"><i class="cf cf-scatter-chart"></i>
|
||||
ng-click="setGraphMode('scatterChart', true)"
|
||||
tooltip="Scatter Chart" tooltip-placement="bottom"><i class="cf cf-scatter-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('map')}"
|
||||
ng-click="setGraphMode('map', true)"
|
||||
tooltip="Map" tooltip-placement="bottom"><i class="fa fa-map-marker"></i>
|
||||
</button>
|
||||
|
||||
<button type="button"
|
||||
|
|
|
|||
|
|
@ -51,4 +51,12 @@ limitations under the License.
|
|||
id="p{{paragraph.id}}_scatterChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='map'" id="p{{paragraph.id}}_map"
|
||||
ng-switch="paragraph.config.graph.map.isOnline">
|
||||
<div ng-switch-when="true"></div>
|
||||
<span class="map-offline-text" ng-switch-default>
|
||||
<span>Maps require internet connectivity.</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -28,4 +28,19 @@ limitations under the License.
|
|||
show line chart with focus
|
||||
</label>
|
||||
</div>
|
||||
<div ng-if="isGraphMode('map')">
|
||||
<label>Basemap</label>
|
||||
<span class="dropdown">
|
||||
<button type="button" class="btn btn-default btn-sm dropdown-toggle" style="min-width: 0px;" data-toggle="dropdown">
|
||||
<span ng-bind="paragraph.config.graph.map.baseMapType"></span>
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu" role="menu" style="min-width: 70px;">
|
||||
<li ng-repeat="opt in baseMapOption">
|
||||
<a ng-click="setMapBaseMap(opt)">{{opt}}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
<br/>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ limitations under the License.
|
|||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="row" ng-if="getGraphMode()!='scatterChart'">
|
||||
<div class="row" ng-if="getGraphMode()!='scatterChart' && getGraphMode()!='map'">
|
||||
<div class="col-md-4">
|
||||
<span class="columns lightBold">
|
||||
Keys
|
||||
|
|
@ -165,4 +165,52 @@ limitations under the License.
|
|||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row" ng-if="getGraphMode()=='map'">
|
||||
<div class="col-md-4">
|
||||
<span class="columns lightBold">
|
||||
Latitude
|
||||
<ul data-drop="true"
|
||||
ng-model="paragraph.config.graph.map.lat"
|
||||
jqyoui-droppable="{onDrop:'onGraphOptionChange()'}"
|
||||
class="list-unstyled">
|
||||
<li ng-if="paragraph.config.graph.map.lat">
|
||||
<div class="btn btn-primary btn-xs">
|
||||
{{paragraph.config.graph.map.lat.name}} <span class="fa fa-close" ng-click="removeMapOptionLat($index)"></span>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<span class="columns lightBold">
|
||||
Longitude
|
||||
<ul data-drop="true"
|
||||
ng-model="paragraph.config.graph.map.lng"
|
||||
jqyoui-droppable="{onDrop:'onGraphOptionChange()'}"
|
||||
class="list-unstyled">
|
||||
<li ng-if="paragraph.config.graph.map.lng">
|
||||
<div class="btn btn-primary btn-xs">
|
||||
{{paragraph.config.graph.map.lng.name}} <span class="fa fa-close" ng-click="removeMapOptionLng($index)"></span>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<span class="columns lightBold">
|
||||
Pin contents
|
||||
<ul data-drop="true"
|
||||
ng-model="paragraph.config.graph.map.pinCols"
|
||||
jqyoui-droppable="{multiple:true, onDrop:'onGraphOptionChange()'}"
|
||||
class="list-unstyled">
|
||||
<li ng-repeat="col in paragraph.config.graph.map.pinCols">
|
||||
<div class="btn btn-primary btn-xs">
|
||||
{{col.name}} <span class="fa fa-close" ng-click="removeMapOptionPinInfo($index)"></span>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $rootScope, $route, $window,
|
||||
$routeParams, $location, $timeout, $compile,
|
||||
$http, websocketMsgSrv, baseUrlSrv, ngToast,
|
||||
saveAsService) {
|
||||
saveAsService, esriLoader) {
|
||||
var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_';
|
||||
$scope.parentNote = null;
|
||||
$scope.paragraph = null;
|
||||
|
|
@ -93,12 +93,12 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$scope.parentNote = note;
|
||||
$scope.originalText = angular.copy(newParagraph.text);
|
||||
$scope.chart = {};
|
||||
$scope.baseMapOption = ['Streets', 'Satellite', 'Hybrid', 'Topo', 'Gray', 'Oceans', 'Terrain'];
|
||||
$scope.colWidthOption = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
|
||||
$scope.paragraphFocused = false;
|
||||
if (newParagraph.focus) {
|
||||
$scope.paragraphFocused = true;
|
||||
}
|
||||
|
||||
if (!$scope.paragraph.config) {
|
||||
$scope.paragraph.config = {};
|
||||
}
|
||||
|
|
@ -245,6 +245,22 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
config.graph.scatter = {};
|
||||
}
|
||||
|
||||
if (!config.graph.map) {
|
||||
config.graph.map = {};
|
||||
}
|
||||
|
||||
if (!config.graph.map.baseMapType) {
|
||||
config.graph.map.baseMapType = $scope.baseMapOption[0];
|
||||
}
|
||||
|
||||
if (!config.graph.map.isOnline) {
|
||||
config.graph.map.isOnline = true;
|
||||
}
|
||||
|
||||
if (!config.graph.map.pinCols) {
|
||||
config.graph.map.pinCols = [];
|
||||
}
|
||||
|
||||
if (config.enabled === undefined) {
|
||||
config.enabled = true;
|
||||
}
|
||||
|
|
@ -539,9 +555,10 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$scope.editor.setHighlightGutterLine(false);
|
||||
$scope.editor.getSession().setUseWrapMode(true);
|
||||
$scope.editor.setTheme('ace/theme/chrome');
|
||||
$scope.editor.setReadOnly($scope.isRunning());
|
||||
if ($scope.paragraphFocused) {
|
||||
$scope.editor.focus();
|
||||
$scope.goToLineEnd();
|
||||
$scope.goToEnd();
|
||||
}
|
||||
|
||||
autoAdjustEditorHeight(_editor.container.id);
|
||||
|
|
@ -822,8 +839,8 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
return false;
|
||||
};
|
||||
|
||||
$scope.goToLineEnd = function() {
|
||||
$scope.editor.navigateLineEnd();
|
||||
$scope.goToEnd = function() {
|
||||
$scope.editor.navigateFileEnd();
|
||||
};
|
||||
|
||||
$scope.getResultType = function(paragraph) {
|
||||
|
|
@ -907,6 +924,8 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
|
||||
if (!type || type === 'table') {
|
||||
setTable($scope.paragraph.result, refresh);
|
||||
} else if (type === 'map') {
|
||||
setMap($scope.paragraph.result, refresh);
|
||||
} else {
|
||||
setD3Chart(type, $scope.paragraph.result, refresh);
|
||||
}
|
||||
|
|
@ -1034,7 +1053,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
d3g = scatterData.d3g;
|
||||
|
||||
$scope.chart[type].xAxis.tickFormat(function(d) {return xAxisTickFormat(d, xLabels);});
|
||||
$scope.chart[type].yAxis.tickFormat(function(d) {return xAxisTickFormat(d, yLabels);});
|
||||
$scope.chart[type].yAxis.tickFormat(function(d) {return yAxisTickFormat(d, yLabels);});
|
||||
|
||||
// configure how the tooltip looks.
|
||||
$scope.chart[type].tooltipContent(function(key, x, y, graph, data) {
|
||||
|
|
@ -1076,7 +1095,11 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
xLabels = pivotdata.xLabels;
|
||||
d3g = pivotdata.d3g;
|
||||
$scope.chart[type].xAxis.tickFormat(function(d) {return xAxisTickFormat(d, xLabels);});
|
||||
$scope.chart[type].yAxis.tickFormat(function(d) {return yAxisTickFormat(d);});
|
||||
if (type === 'stackedAreaChart') {
|
||||
$scope.chart[type].yAxisTickFormat(function(d) {return yAxisTickFormat(d);});
|
||||
} else {
|
||||
$scope.chart[type].yAxis.tickFormat(function(d) {return yAxisTickFormat(d, xLabels);});
|
||||
}
|
||||
$scope.chart[type].yAxis.axisLabelDistance(50);
|
||||
if ($scope.chart[type].useInteractiveGuideline) { // lineWithFocusChart hasn't got useInteractiveGuideline
|
||||
$scope.chart[type].useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)
|
||||
|
|
@ -1131,6 +1154,236 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$timeout(retryRenderer);
|
||||
};
|
||||
|
||||
var setMap = function(data, refresh) {
|
||||
var createPinMapLayer = function(pins, cb) {
|
||||
esriLoader.require(['esri/layers/FeatureLayer'], function(FeatureLayer) {
|
||||
var pinLayer = new FeatureLayer({
|
||||
id: 'pins',
|
||||
spatialReference: $scope.map.spatialReference,
|
||||
geometryType: 'point',
|
||||
source: pins,
|
||||
fields: [],
|
||||
objectIdField: '_ObjectID',
|
||||
renderer: $scope.map.pinRenderer,
|
||||
popupTemplate: {
|
||||
title: '[{_lng}, {_lat}]',
|
||||
content: [{
|
||||
type: 'fields',
|
||||
fieldInfos: []
|
||||
}]
|
||||
}
|
||||
});
|
||||
|
||||
// add user-selected pin info fields to popup
|
||||
var pinInfoCols = $scope.paragraph.config.graph.map.pinCols;
|
||||
for (var i = 0; i < pinInfoCols.length; ++i) {
|
||||
pinLayer.popupTemplate.content[0].fieldInfos.push({
|
||||
fieldName: pinInfoCols[i].name,
|
||||
visible: true
|
||||
});
|
||||
}
|
||||
cb(pinLayer);
|
||||
});
|
||||
};
|
||||
|
||||
var getMapPins = function(cb) {
|
||||
esriLoader.require(['esri/geometry/Point'], function(Point, FeatureLayer) {
|
||||
var latCol = $scope.paragraph.config.graph.map.lat;
|
||||
var lngCol = $scope.paragraph.config.graph.map.lng;
|
||||
var pinInfoCols = $scope.paragraph.config.graph.map.pinCols;
|
||||
var pins = [];
|
||||
|
||||
// construct objects for pins
|
||||
if (latCol && lngCol && data.rows) {
|
||||
for (var i = 0; i < data.rows.length; ++i) {
|
||||
var row = data.rows[i];
|
||||
var lng = row[lngCol.index];
|
||||
var lat = row[latCol.index];
|
||||
var pin = {
|
||||
geometry: new Point({
|
||||
longitude: lng,
|
||||
latitude: lat,
|
||||
spatialReference: $scope.map.spatialReference
|
||||
}),
|
||||
attributes: {
|
||||
_ObjectID: i,
|
||||
_lng: lng,
|
||||
_lat: lat
|
||||
}
|
||||
};
|
||||
|
||||
// add pin info from user-selected columns
|
||||
for (var j = 0; j < pinInfoCols.length; ++j) {
|
||||
var col = pinInfoCols[j];
|
||||
pin.attributes[col.name] = row[col.index];
|
||||
}
|
||||
pins.push(pin);
|
||||
}
|
||||
}
|
||||
cb(pins);
|
||||
});
|
||||
};
|
||||
|
||||
var updateMapPins = function() {
|
||||
var pinLayer = $scope.map.map.findLayerById('pins');
|
||||
$scope.map.popup.close();
|
||||
if (pinLayer) {
|
||||
$scope.map.map.remove(pinLayer);
|
||||
}
|
||||
|
||||
// add pins to map as layer
|
||||
getMapPins(function(pins) {
|
||||
createPinMapLayer(pins, function(pinLayer) {
|
||||
$scope.map.map.add(pinLayer);
|
||||
if (pinLayer.source.length > 0) {
|
||||
$scope.map.goTo(pinLayer.source);
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
var createMap = function(mapdiv) {
|
||||
// prevent zooming with the scroll wheel
|
||||
var disableZoom = function(e) {
|
||||
var evt = e || window.event;
|
||||
evt.cancelBubble = true;
|
||||
evt.returnValue = false;
|
||||
if (evt.stopPropagation) {
|
||||
evt.stopPropagation();
|
||||
}
|
||||
};
|
||||
var eName = window.WheelEvent ? 'wheel' : // Modern browsers
|
||||
window.MouseWheelEvent ? 'mousewheel' : // WebKit and IE
|
||||
'DOMMouseScroll'; // Old Firefox
|
||||
mapdiv.addEventListener(eName, disableZoom, true);
|
||||
|
||||
esriLoader.require(['esri/views/MapView',
|
||||
'esri/Map',
|
||||
'esri/renderers/SimpleRenderer',
|
||||
'esri/symbols/SimpleMarkerSymbol'],
|
||||
function(MapView, Map, SimpleRenderer, SimpleMarkerSymbol) {
|
||||
$scope.map = new MapView({
|
||||
container: mapdiv,
|
||||
map: new Map({
|
||||
basemap: $scope.paragraph.config.graph.map.baseMapType.toLowerCase()
|
||||
}),
|
||||
center: [-106.3468, 56.1304], // Canada (lng, lat)
|
||||
zoom: 2,
|
||||
pinRenderer: new SimpleRenderer({
|
||||
symbol: new SimpleMarkerSymbol({
|
||||
'color': [255, 0, 0, 0.5],
|
||||
'size': 16.5,
|
||||
'outline': {
|
||||
'color': [0, 0, 0, 1],
|
||||
'width': 1.125,
|
||||
},
|
||||
// map pin SVG path
|
||||
'path': 'M16,3.5c-4.142,0-7.5,3.358-7.5,7.5c0,4.143,7.5,18.121,7.5,' +
|
||||
'18.121S23.5,15.143,23.5,11C23.5,6.858,20.143,3.5,16,3.5z ' +
|
||||
'M16,14.584c-1.979,0-3.584-1.604-3.584-3.584S14.021,7.416,' +
|
||||
'16,7.416S19.584,9.021,19.584,11S17.979,14.584,16,14.584z'
|
||||
})
|
||||
})
|
||||
});
|
||||
|
||||
$scope.map.on('click', function() {
|
||||
// ArcGIS JS API 4.0 does not account for scrolling or position
|
||||
// changes by default (this is a bug, to be fixed in the upcoming
|
||||
// version 4.1; see https://geonet.esri.com/thread/177238#comment-609681).
|
||||
// This results in a misaligned popup.
|
||||
|
||||
// Workaround: manually set popup position to match position of selected pin
|
||||
if ($scope.map.popup.selectedFeature) {
|
||||
$scope.map.popup.location = $scope.map.popup.selectedFeature.geometry;
|
||||
}
|
||||
});
|
||||
$scope.map.then(updateMapPins);
|
||||
});
|
||||
};
|
||||
|
||||
var checkMapOnline = function(cb) {
|
||||
// are we able to get a response from the ArcGIS servers?
|
||||
var callback = function(res) {
|
||||
var online = (res.status > 0);
|
||||
$scope.paragraph.config.graph.map.isOnline = online;
|
||||
cb(online);
|
||||
};
|
||||
$http.head('//services.arcgisonline.com/arcgis/', {
|
||||
timeout: 5000,
|
||||
withCredentials: false
|
||||
}).then(callback, callback);
|
||||
};
|
||||
|
||||
var renderMap = function() {
|
||||
var mapdiv = angular.element('#p' + $scope.paragraph.id + '_map')
|
||||
.css('height', $scope.paragraph.config.graph.height)
|
||||
.children('div').get(0);
|
||||
|
||||
// on chart type change, destroy map to force reinitialization.
|
||||
if ($scope.map && !refresh) {
|
||||
$scope.map.map.destroy();
|
||||
$scope.map.pinRenderer = null;
|
||||
$scope.map = null;
|
||||
}
|
||||
|
||||
var requireMapCSS = function() {
|
||||
var url = '//js.arcgis.com/4.0/esri/css/main.css';
|
||||
if (!angular.element('link[href="' + url + '"]').length) {
|
||||
var link = document.createElement('link');
|
||||
link.rel = 'stylesheet';
|
||||
link.type = 'text/css';
|
||||
link.href = url;
|
||||
angular.element('head').append(link);
|
||||
}
|
||||
};
|
||||
|
||||
var requireMapJS = function(cb) {
|
||||
if (!esriLoader.isLoaded()) {
|
||||
esriLoader.bootstrap({
|
||||
url: '//js.arcgis.com/4.0'
|
||||
}).then(cb);
|
||||
} else {
|
||||
cb();
|
||||
}
|
||||
};
|
||||
|
||||
checkMapOnline(function(online) {
|
||||
// we need an internet connection to use the map
|
||||
if (online) {
|
||||
// create map if not exists.
|
||||
if (!$scope.map) {
|
||||
requireMapCSS();
|
||||
requireMapJS(function() {
|
||||
createMap(mapdiv);
|
||||
});
|
||||
} else {
|
||||
updateMapPins();
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
var retryRenderer = function() {
|
||||
if (angular.element('#p' + $scope.paragraph.id + '_map div').length) {
|
||||
try {
|
||||
renderMap();
|
||||
} catch (err) {
|
||||
console.log('Map drawing error %o', err);
|
||||
}
|
||||
} else {
|
||||
$timeout(retryRenderer,10);
|
||||
}
|
||||
};
|
||||
$timeout(retryRenderer);
|
||||
};
|
||||
|
||||
$scope.setMapBaseMap = function(bm) {
|
||||
$scope.paragraph.config.graph.map.baseMapType = bm;
|
||||
if ($scope.map) {
|
||||
$scope.map.map.basemap = bm.toLowerCase();
|
||||
}
|
||||
};
|
||||
|
||||
$scope.isGraphMode = function(graphName) {
|
||||
var activeAppId = _.get($scope.paragraph.config, 'helium.activeApp');
|
||||
if ($scope.getResultType() === 'TABLE' && $scope.getGraphMode() === graphName && !activeAppId) {
|
||||
|
|
@ -1193,6 +1446,24 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$scope.setGraphMode($scope.paragraph.config.graph.mode, true, false);
|
||||
};
|
||||
|
||||
$scope.removeMapOptionLat = function(idx) {
|
||||
$scope.paragraph.config.graph.map.lat = null;
|
||||
clearUnknownColsFromGraphOption();
|
||||
$scope.setGraphMode($scope.paragraph.config.graph.mode, true, false);
|
||||
};
|
||||
|
||||
$scope.removeMapOptionLng = function(idx) {
|
||||
$scope.paragraph.config.graph.map.lng = null;
|
||||
clearUnknownColsFromGraphOption();
|
||||
$scope.setGraphMode($scope.paragraph.config.graph.mode, true, false);
|
||||
};
|
||||
|
||||
$scope.removeMapOptionPinInfo = function(idx) {
|
||||
$scope.paragraph.config.graph.map.pinCols.splice(idx, 1);
|
||||
clearUnknownColsFromGraphOption();
|
||||
$scope.setGraphMode($scope.paragraph.config.graph.mode, true, false);
|
||||
};
|
||||
|
||||
/* Clear unknown columns from graph option */
|
||||
var clearUnknownColsFromGraphOption = function() {
|
||||
var unique = function(list) {
|
||||
|
|
@ -1223,7 +1494,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
}
|
||||
};
|
||||
|
||||
var removeUnknownFromScatterSetting = function(fields) {
|
||||
var removeUnknownFromFields = function(fields) {
|
||||
for (var f in fields) {
|
||||
if (fields[f]) {
|
||||
var found = false;
|
||||
|
|
@ -1235,7 +1506,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
if (!found && (fields[f] instanceof Object) && !(fields[f] instanceof Array)) {
|
||||
fields[f] = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1250,7 +1521,11 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
unique($scope.paragraph.config.graph.groups);
|
||||
removeUnknown($scope.paragraph.config.graph.groups);
|
||||
|
||||
removeUnknownFromScatterSetting($scope.paragraph.config.graph.scatter);
|
||||
removeUnknownFromFields($scope.paragraph.config.graph.scatter);
|
||||
|
||||
unique($scope.paragraph.config.graph.map.pinCols);
|
||||
removeUnknown($scope.paragraph.config.graph.map.pinCols);
|
||||
removeUnknownFromFields($scope.paragraph.config.graph.map);
|
||||
};
|
||||
|
||||
/* select default key and value if there're none selected */
|
||||
|
|
@ -1271,6 +1546,23 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$scope.paragraph.config.graph.scatter.xAxis = $scope.paragraph.result.columnNames[0];
|
||||
}
|
||||
}
|
||||
|
||||
/* try to find columns for the map logitude and latitude */
|
||||
var findDefaultMapCol = function(settingName, keyword) {
|
||||
var col;
|
||||
if (!$scope.paragraph.config.graph.map[settingName]) {
|
||||
for (var i = 0; i < $scope.paragraph.result.columnNames.length; ++i) {
|
||||
col = $scope.paragraph.result.columnNames[i];
|
||||
if (col.name.toUpperCase().indexOf(keyword) !== -1) {
|
||||
$scope.paragraph.config.graph.map[settingName] = col;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
findDefaultMapCol('lat', 'LAT');
|
||||
findDefaultMapCol('lng', 'LONG');
|
||||
};
|
||||
|
||||
var pivot = function(data) {
|
||||
|
|
@ -1544,7 +1836,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
if (groups.length === 1 && values.length === 1) {
|
||||
for (d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
|
||||
colName = d3g[d3gIndex].key;
|
||||
colName = colName.split('.')[0];
|
||||
colName = colName.split('.').slice(0, -1).join('.');
|
||||
d3g[d3gIndex].key = colName;
|
||||
}
|
||||
}
|
||||
|
|
@ -2228,6 +2520,7 @@ angular.module('zeppelinWebApp').controller('ParagraphCtrl', function($scope, $r
|
|||
$scope.paragraph.status = data.paragraph.status;
|
||||
$scope.paragraph.result = data.paragraph.result;
|
||||
$scope.paragraph.settings = data.paragraph.settings;
|
||||
$scope.editor.setReadOnly($scope.isRunning());
|
||||
|
||||
if (!$scope.asIframe) {
|
||||
$scope.paragraph.config = data.paragraph.config;
|
||||
|
|
|
|||
|
|
@ -275,6 +275,10 @@ table.dataTable.table-condensed .sorting_desc:after {
|
|||
background: none !important;
|
||||
}
|
||||
|
||||
.paragraph-disable {
|
||||
opacity : 0.6!important;
|
||||
}
|
||||
|
||||
.ace_marker-layer .ace_selection {
|
||||
z-index: 0 !important;
|
||||
}
|
||||
|
|
@ -327,12 +331,41 @@ table.dataTable.table-condensed .sorting_desc:after {
|
|||
.tableDisplay div {
|
||||
}
|
||||
|
||||
.tableDisplay img {
|
||||
.tableDisplay img:not(.esri-bitmap) {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
}
|
||||
|
||||
.esri-display-object > svg {
|
||||
overflow: visible;
|
||||
}
|
||||
|
||||
.esri-popup > .esri-docked.esri-dock-to-bottom {
|
||||
padding: 8px;
|
||||
margin-top: 0px;
|
||||
}
|
||||
|
||||
.esri-popup-main {
|
||||
max-height: 100%;
|
||||
}
|
||||
|
||||
span.map-offline-text {
|
||||
display: table;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
span.map-offline-text > span {
|
||||
display: table-cell;
|
||||
vertical-align: middle;
|
||||
font-size: 18px;
|
||||
font-weight: 700;
|
||||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
color: #212121;
|
||||
}
|
||||
|
||||
.tableDisplay .btn-group span {
|
||||
margin: 10px 0 0 10px;
|
||||
font-size: 12px;
|
||||
|
|
@ -350,7 +383,8 @@ table.dataTable.table-condensed .sorting_desc:after {
|
|||
|
||||
}
|
||||
|
||||
.tableDisplay .option .columns {
|
||||
.tableDisplay .option .columns,
|
||||
div.esri-view {
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,16 +38,14 @@ limitations under the License.
|
|||
<div>
|
||||
<div ng-show="!paragraph.config.editorHide && !viewOnly" style="margin-bottom:3px;">
|
||||
<div id="{{paragraph.id}}_editor"
|
||||
style="opacity: 1;"
|
||||
class="editor"
|
||||
ui-ace="{
|
||||
onLoad : aceLoaded,
|
||||
require : ['ace/ext/language_tools']
|
||||
}"
|
||||
ng-model="paragraph.text"
|
||||
ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING',
|
||||
'paragraph-text--dirty' : dirtyText !== originalText && dirtyText !== undefined}">
|
||||
</div>
|
||||
ng-class="{'paragraph-disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING',
|
||||
'paragraph-text--dirty' : dirtyText !== originalText && dirtyText !== undefined}"> </div>
|
||||
</div>
|
||||
|
||||
<div ng-include src="'app/notebook/paragraph/paragraph-progressBar.html'"></div>
|
||||
|
|
|
|||
|
|
@ -17,9 +17,11 @@ angular.module('zeppelinWebApp').directive('ngEnter', function() {
|
|||
return function(scope, element, attrs) {
|
||||
element.bind('keydown keypress', function(event) {
|
||||
if (event.which === 13) {
|
||||
scope.$apply(function() {
|
||||
scope.$eval(attrs.ngEnter);
|
||||
});
|
||||
if (!event.shiftKey) {
|
||||
scope.$apply(function() {
|
||||
scope.$eval(attrs.ngEnter);
|
||||
});
|
||||
}
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ limitations under the License.
|
|||
<script src="bower_components/handsontable/dist/handsontable.js"></script>
|
||||
<script src="bower_components/moment-duration-format/lib/moment-duration-format.js"></script>
|
||||
<script src="bower_components/select2/dist/js/select2.js"></script>
|
||||
<script src="bower_components/angular-esri-map/dist/angular-esri-map.js"></script>
|
||||
<!-- endbower -->
|
||||
<!-- endbuild -->
|
||||
<!-- build:js({.tmp,src}) scripts/scripts.js -->
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ module.exports = function(config) {
|
|||
'bower_components/handsontable/dist/handsontable.js',
|
||||
'bower_components/moment-duration-format/lib/moment-duration-format.js',
|
||||
'bower_components/select2/dist/js/select2.js',
|
||||
'bower_components/angular-esri-map/dist/angular-esri-map.js',
|
||||
'bower_components/angular-mocks/angular-mocks.js',
|
||||
// endbower
|
||||
'src/app/app.js',
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ describe('Controller: ParagraphCtrl', function() {
|
|||
'getResultType', 'loadTableData', 'setGraphMode', 'isGraphMode', 'onGraphOptionChange',
|
||||
'removeGraphOptionKeys', 'removeGraphOptionValues', 'removeGraphOptionGroups', 'setGraphOptionValueAggr',
|
||||
'removeScatterOptionXaxis', 'removeScatterOptionYaxis', 'removeScatterOptionGroup',
|
||||
'removeScatterOptionSize'];
|
||||
'removeScatterOptionSize', 'removeMapOptionLat', 'removeMapOptionLng', 'removeMapOptionPinInfo'];
|
||||
|
||||
functions.forEach(function(fn) {
|
||||
it('check for scope functions to be defined : ' + fn, function() {
|
||||
|
|
|
|||
|
|
@ -97,13 +97,13 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
* This is only references with default settings, name and properties
|
||||
* key: InterpreterSetting.name
|
||||
*/
|
||||
private Map<String, InterpreterSetting> interpreterSettingsRef = new HashMap<>();
|
||||
private final Map<String, InterpreterSetting> interpreterSettingsRef = new HashMap<>();
|
||||
|
||||
/**
|
||||
* This is used by creating and running Interpreters
|
||||
* key: InterpreterSetting.id <- This is becuase backward compatibility
|
||||
*/
|
||||
private Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
|
||||
private final Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
|
||||
|
||||
private Map<String, List<String>> interpreterBindings = new HashMap<>();
|
||||
private List<RemoteRepository> interpreterRepositories;
|
||||
|
|
@ -175,7 +175,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
|
||||
registerInterpreterFromResource(cl, interpreterDirString, interpreterJson);
|
||||
|
||||
/**
|
||||
/*
|
||||
* TODO(jongyoul)
|
||||
* - Remove these codes below because of legacy code
|
||||
* - Support ThreadInterpreter
|
||||
|
|
@ -468,8 +468,6 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
* Return ordered interpreter setting list.
|
||||
* The list does not contain more than one setting from the same interpreter class.
|
||||
* Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public List<String> getDefaultInterpreterSettingList() {
|
||||
// this list will contain default interpreter setting list
|
||||
|
|
@ -529,8 +527,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param group InterpreterSetting reference name
|
||||
* @param properties
|
||||
* @param group InterpreterSetting reference name
|
||||
* @return
|
||||
*/
|
||||
public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
|
||||
|
|
@ -579,8 +576,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
|
||||
} else {
|
||||
interpreterSetting =
|
||||
new InterpreterSetting(group, null, interpreterInfos, properties, dependencies,
|
||||
option, path);
|
||||
new InterpreterSetting(group, null, interpreterInfos, properties, dependencies, option,
|
||||
path);
|
||||
interpreterSettingsRef.put(group, interpreterSetting);
|
||||
}
|
||||
}
|
||||
|
|
@ -1176,6 +1173,16 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
return interpreters.get(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Support the legacy way to use it
|
||||
for (InterpreterSetting s : settings) {
|
||||
if (s.getGroup().equals(replName)) {
|
||||
List<Interpreter> interpreters = createOrGetInterpreterList(noteId, s);
|
||||
if (null != interpreters) {
|
||||
return interpreters.get(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dev interpreter
|
||||
|
|
|
|||
|
|
@ -411,24 +411,40 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
List<Map<String, String>> paragraphsInfo = new LinkedList<>();
|
||||
synchronized (paragraphs) {
|
||||
for (Paragraph p : paragraphs) {
|
||||
Map<String, String> info = new HashMap<>();
|
||||
info.put("id", p.getId());
|
||||
info.put("status", p.getStatus().toString());
|
||||
if (p.getDateStarted() != null) {
|
||||
info.put("started", p.getDateStarted().toString());
|
||||
}
|
||||
if (p.getDateFinished() != null) {
|
||||
info.put("finished", p.getDateFinished().toString());
|
||||
}
|
||||
if (p.getStatus().isRunning()) {
|
||||
info.put("progress", String.valueOf(p.progress()));
|
||||
}
|
||||
Map<String, String> info = populatePragraphInfo(p);
|
||||
paragraphsInfo.add(info);
|
||||
}
|
||||
}
|
||||
return paragraphsInfo;
|
||||
}
|
||||
|
||||
public Map<String, String> generateSingleParagraphInfo(String paragraphId) {
|
||||
synchronized (paragraphs) {
|
||||
for (Paragraph p : paragraphs) {
|
||||
if (p.getId().equals(paragraphId)) {
|
||||
return populatePragraphInfo(p);
|
||||
}
|
||||
}
|
||||
return new HashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> populatePragraphInfo(Paragraph p) {
|
||||
Map<String, String> info = new HashMap<>();
|
||||
info.put("id", p.getId());
|
||||
info.put("status", p.getStatus().toString());
|
||||
if (p.getDateStarted() != null) {
|
||||
info.put("started", p.getDateStarted().toString());
|
||||
}
|
||||
if (p.getDateFinished() != null) {
|
||||
info.put("finished", p.getDateFinished().toString());
|
||||
}
|
||||
if (p.getStatus().isRunning()) {
|
||||
info.put("progress", String.valueOf(p.progress()));
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all paragraphs sequentially.
|
||||
*/
|
||||
|
|
@ -445,11 +461,7 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
AuthenticationInfo authenticationInfo = new AuthenticationInfo();
|
||||
authenticationInfo.setUser(cronExecutingUser);
|
||||
p.setAuthenticationInfo(authenticationInfo);
|
||||
|
||||
p.setListener(jobListenerFactory.getParagraphJobListener(this));
|
||||
Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName());
|
||||
|
||||
intp.getScheduler().submit(p);
|
||||
run(p.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -472,7 +484,18 @@ public class Note implements Serializable, ParagraphJobListener {
|
|||
logger.debug("New paragraph: {}", pText);
|
||||
p.setEffectiveText(pText);
|
||||
} else {
|
||||
throw new InterpreterException("Interpreter " + requiredReplName + " not found");
|
||||
String intpExceptionMsg = String.format("%s",
|
||||
p.getJobName()
|
||||
+ "'s Interpreter "
|
||||
+ requiredReplName + " not found"
|
||||
);
|
||||
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
|
||||
InterpreterResult intpResult = new InterpreterResult(
|
||||
InterpreterResult.Code.ERROR, intpException.getMessage()
|
||||
);
|
||||
p.setReturn(intpResult, intpException);
|
||||
p.setStatus(Job.Status.ERROR);
|
||||
throw intpException;
|
||||
}
|
||||
}
|
||||
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ import org.apache.zeppelin.user.AuthenticationInfo;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
|
||||
|
|
@ -191,20 +193,45 @@ public class ZeppelinHubRepo implements NotebookRepo {
|
|||
@Override
|
||||
public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
|
||||
throws IOException {
|
||||
// Auto-generated method stub
|
||||
return null;
|
||||
if (StringUtils.isBlank(noteId)) {
|
||||
return null;
|
||||
}
|
||||
String endpoint = Joiner.on("/").join(noteId, "checkpoint");
|
||||
String content = GSON.toJson(ImmutableMap.of("message", checkpointMsg));
|
||||
String response = restApiClient.asyncPutWithResponseBody(endpoint, content);
|
||||
|
||||
return GSON.fromJson(response, Revision.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException {
|
||||
// Auto-generated method stub
|
||||
return null;
|
||||
if (StringUtils.isBlank(noteId) || StringUtils.isBlank(revId)) {
|
||||
return EMPTY_NOTE;
|
||||
}
|
||||
String endpoint = Joiner.on("/").join(noteId, "checkpoint", revId);
|
||||
String response = restApiClient.asyncGet(endpoint);
|
||||
Note note = GSON.fromJson(response, Note.class);
|
||||
if (note == null) {
|
||||
return EMPTY_NOTE;
|
||||
}
|
||||
LOG.info("ZeppelinHub REST API get note {} revision {}", noteId, revId);
|
||||
return note;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
|
||||
// Auto-generated method stub
|
||||
return null;
|
||||
if (StringUtils.isBlank(noteId)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
String endpoint = Joiner.on("/").join(noteId, "checkpoint");
|
||||
List<Revision> history = Collections.emptyList();
|
||||
try {
|
||||
String response = restApiClient.asyncGet(endpoint);
|
||||
history = GSON.fromJson(response, new TypeToken<List<Revision>>(){}.getType());
|
||||
} catch (IOException e) {
|
||||
LOG.error("Cannot get note history", e);
|
||||
}
|
||||
return history;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,9 +25,8 @@ import java.util.concurrent.TimeoutException;
|
|||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.eclipse.jetty.client.HttpClient;
|
||||
import org.eclipse.jetty.client.api.Request;
|
||||
import org.eclipse.jetty.client.api.Response;
|
||||
import org.eclipse.jetty.client.api.Result;
|
||||
import org.eclipse.jetty.client.util.BufferingResponseListener;
|
||||
import org.eclipse.jetty.client.util.InputStreamResponseListener;
|
||||
import org.eclipse.jetty.client.util.StringContentProvider;
|
||||
import org.eclipse.jetty.http.HttpMethod;
|
||||
|
|
@ -115,89 +114,66 @@ public class ZeppelinhubRestApiHandler {
|
|||
}
|
||||
|
||||
public String asyncGet(String argument) throws IOException {
|
||||
String note = StringUtils.EMPTY;
|
||||
|
||||
InputStreamResponseListener listener = new InputStreamResponseListener();
|
||||
client.newRequest(zepelinhubUrl + argument)
|
||||
.header(ZEPPELIN_TOKEN_HEADER, token)
|
||||
.send(listener);
|
||||
|
||||
// Wait for the response headers to arrive
|
||||
Response response;
|
||||
try {
|
||||
response = listener.get(30, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException | TimeoutException | ExecutionException e) {
|
||||
LOG.error("Cannot perform Get request to ZeppelinHub", e);
|
||||
throw new IOException("Cannot load note from ZeppelinHub", e);
|
||||
}
|
||||
|
||||
int code = response.getStatus();
|
||||
if (code == 200) {
|
||||
try (InputStream responseContent = listener.getInputStream()) {
|
||||
note = IOUtils.toString(responseContent, "UTF-8");
|
||||
}
|
||||
} else {
|
||||
LOG.error("ZeppelinHub Get {} returned with status {} ", zepelinhubUrl + argument, code);
|
||||
throw new IOException("Cannot load note from ZeppelinHub");
|
||||
}
|
||||
return note;
|
||||
return sendToZeppelinHub(HttpMethod.GET, zepelinhubUrl + argument);
|
||||
}
|
||||
|
||||
|
||||
public String asyncPutWithResponseBody(String url, String json) throws IOException {
|
||||
if (StringUtils.isBlank(url) || StringUtils.isBlank(json)) {
|
||||
LOG.error("Empty note, cannot send it to zeppelinHub");
|
||||
throw new IOException("Cannot send emtpy note to zeppelinHub");
|
||||
}
|
||||
return sendToZeppelinHub(HttpMethod.PUT, zepelinhubUrl + url, json);
|
||||
}
|
||||
|
||||
public void asyncPut(String jsonNote) throws IOException {
|
||||
if (StringUtils.isBlank(jsonNote)) {
|
||||
LOG.error("Cannot save empty note/string to ZeppelinHub");
|
||||
return;
|
||||
}
|
||||
|
||||
client.newRequest(zepelinhubUrl).method(HttpMethod.PUT)
|
||||
.header(ZEPPELIN_TOKEN_HEADER, token)
|
||||
.content(new StringContentProvider(jsonNote, "UTF-8"), "application/json;charset=UTF-8")
|
||||
.send(new BufferingResponseListener() {
|
||||
|
||||
@Override
|
||||
public void onComplete(Result res) {
|
||||
if (!res.isFailed() && res.getResponse().getStatus() == 200) {
|
||||
LOG.info("Successfully saved note to ZeppelinHub with {}",
|
||||
res.getResponse().getStatus());
|
||||
} else {
|
||||
LOG.warn("Failed to save note to ZeppelinHub with HttpStatus {}",
|
||||
res.getResponse().getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Response response, Throwable failure) {
|
||||
LOG.error("Failed to save note to ZeppelinHub: {}", response.getReason(), failure);
|
||||
}
|
||||
});
|
||||
sendToZeppelinHub(HttpMethod.PUT, zepelinhubUrl, jsonNote);
|
||||
}
|
||||
|
||||
public void asyncDel(String argument) {
|
||||
public void asyncDel(String argument) throws IOException {
|
||||
if (StringUtils.isBlank(argument)) {
|
||||
LOG.error("Cannot delete empty note from ZeppelinHub");
|
||||
return;
|
||||
}
|
||||
client.newRequest(zepelinhubUrl + argument)
|
||||
.method(HttpMethod.DELETE)
|
||||
.header(ZEPPELIN_TOKEN_HEADER, token)
|
||||
.send(new BufferingResponseListener() {
|
||||
sendToZeppelinHub(HttpMethod.DELETE, zepelinhubUrl + argument);
|
||||
}
|
||||
|
||||
private String sendToZeppelinHub(HttpMethod method, String url) throws IOException {
|
||||
return sendToZeppelinHub(method, url, StringUtils.EMPTY);
|
||||
}
|
||||
|
||||
private String sendToZeppelinHub(HttpMethod method, String url, String json) throws IOException {
|
||||
InputStreamResponseListener listener = new InputStreamResponseListener();
|
||||
Response response;
|
||||
String data;
|
||||
|
||||
@Override
|
||||
public void onComplete(Result res) {
|
||||
if (!res.isFailed() && res.getResponse().getStatus() == 200) {
|
||||
LOG.info("Successfully removed note from ZeppelinHub with {}",
|
||||
res.getResponse().getStatus());
|
||||
} else {
|
||||
LOG.warn("Failed to remove note from ZeppelinHub with HttpStatus {}",
|
||||
res.getResponse().getStatus());
|
||||
}
|
||||
}
|
||||
Request request = client.newRequest(url).method(method).header(ZEPPELIN_TOKEN_HEADER, token);
|
||||
if ((method.equals(HttpMethod.PUT) || method.equals(HttpMethod.POST)) &&
|
||||
!StringUtils.isBlank(json)) {
|
||||
request.content(new StringContentProvider(json, "UTF-8"), "application/json;charset=UTF-8");
|
||||
}
|
||||
request.send(listener);
|
||||
|
||||
@Override
|
||||
public void onFailure(Response response, Throwable failure) {
|
||||
LOG.error("Failed to remove note from ZeppelinHub: {}", response.getReason(), failure);
|
||||
}
|
||||
});
|
||||
try {
|
||||
response = listener.get(30, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException | TimeoutException | ExecutionException e) {
|
||||
LOG.error("Cannot perform {} request to ZeppelinHub", method, e);
|
||||
throw new IOException("Cannot perform " + method + " request to ZeppelinHub", e);
|
||||
}
|
||||
|
||||
int code = response.getStatus();
|
||||
if (code == 200) {
|
||||
try (InputStream responseContent = listener.getInputStream()) {
|
||||
data = IOUtils.toString(responseContent, "UTF-8");
|
||||
}
|
||||
} else {
|
||||
LOG.error("ZeppelinHub {} {} returned with status {} ", method, url, code);
|
||||
throw new IOException("Cannot perform " + method + " request to ZeppelinHub");
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
public void close() {
|
||||
|
|
|
|||
|
|
@ -155,5 +155,6 @@ public class InterpreterFactoryTest {
|
|||
}});
|
||||
|
||||
assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName());
|
||||
assertEquals("className1", factory.getInterpreter("note", "group1").getClassName());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue