mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge branch 'master' into jdbc-impersonation
This commit is contained in:
astroshim
commit
79ba25b56d
72 changed files with 4910 additions and 1212 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -1,4 +1,5 @@
|
|||
*.class
|
||||
*.pyc
|
||||
|
||||
# Package Files #
|
||||
*.jar
|
||||
|
|
@ -6,7 +7,8 @@
|
|||
*.ear
|
||||
|
||||
# interpreter
|
||||
/interpreter/
|
||||
/interpreter/*
|
||||
!/interpreter/lib
|
||||
|
||||
# interpreter temp files
|
||||
spark/derby.log
|
||||
|
|
|
|||
|
|
@ -45,6 +45,10 @@ user3 = password4, role2
|
|||
#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
|
||||
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
|
||||
|
||||
### A sample PAM configuration
|
||||
#pamRealm=org.apache.zeppelin.realm.PamRealm
|
||||
#pamRealm.service=sshd
|
||||
|
||||
### A sample for configuring ZeppelinHub Realm
|
||||
#zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
|
||||
## Url of ZeppelinHub
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
BIN
docs/assets/themes/zeppelin/img/screenshots/display_formula.png
Normal file
BIN
docs/assets/themes/zeppelin/img/screenshots/display_formula.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 78 KiB |
|
|
@ -39,6 +39,13 @@ With `%html` directive, Zeppelin treats your output as HTML
|
|||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/display_html.png" />
|
||||
|
||||
### Mathematical expressions
|
||||
HTML display system automatically formats mathematical expression using [MathJax](https://www.mathjax.org/). You can use
|
||||
`\\( INLINE EXPRESSION \\)` and `$$ EXPRESSION $$` to format. For example
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/display_formula.png" />
|
||||
|
||||
|
||||
## Table
|
||||
|
||||
If you have data that row separated by '\n' (newline) and column separated by '\t' (tab) with first row as header row, for example
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ You can use Zeppelin to retrieve distributed data from cache using Ignite SQL in
|
|||
## Installing and Running Ignite example
|
||||
In order to use Ignite interpreters, you may install Apache Ignite in some simple steps:
|
||||
|
||||
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zeppelin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
|
||||
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
|
||||
1. Ignite provides examples only with source or binary release. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. The supported Ignite version is specified in [Supported Interpreter table](https://zeppelin.apache.org/supported_interpreters.html#ignite) for each Zeppelin release. If you're using Zeppelin master branch, please see `ignite.version` in `path/to/your-Zeppelin/ignite/pom.xml`.
|
||||
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided `<dest_dir>/apache-ignite-fabric-{version}-bin/examples/pom.xml` file into your favourite IDE, such as Eclipse.
|
||||
|
||||
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
|
||||
* Set examples directory path to Eclipse and select the pom.xml.
|
||||
|
|
@ -58,12 +58,12 @@ At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Z
|
|||
<tr>
|
||||
<td>ignite.addresses</td>
|
||||
<td>127.0.0.1:47500..47509</td>
|
||||
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/v1.2/docs/cluster-config) section for more details.</td>
|
||||
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/docs/cluster-config) section for more details.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ignite.clientMode</td>
|
||||
<td>true</td>
|
||||
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/v1.2/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
|
||||
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ignite.config.url</td>
|
||||
|
|
@ -78,7 +78,7 @@ At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Z
|
|||
<tr>
|
||||
<td>ignite.peerClassLoadingEnabled</td>
|
||||
<td>true</td>
|
||||
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/v1.2/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
|
||||
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ After configuring Ignite interpreter, create your own notebook. Then you can bin
|
|||
|
||||
|
||||
|
||||
For more interpreter binding information see [here](http://zeppelin.apache.org/docs/manual/interpreters.html).
|
||||
For more interpreter binding information see [here](../manual/interpreters.html#what-is-interpreter-setting).
|
||||
|
||||
### Ignite SQL interpreter
|
||||
In order to execute SQL query, use ` %ignite.ignitesql ` prefix. <br>
|
||||
|
|
|
|||
220
docs/interpreter/mahout.md
Normal file
220
docs/interpreter/mahout.md
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Mahout Interpreter for Apache Zeppelin"
|
||||
description: "Apache Mahout provides a unified API (the R-Like Scala DSL) for quickly creating machine learning algorithms on a variety of engines."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Apache Mahout Interpreter for Apache Zeppelin
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
## Installation
|
||||
|
||||
Apache Mahout is a collection of packages that enable machine learning and matrix algebra on underlying engines such as Apache Flink or Apache Spark. A convenience script for creating and configuring two Mahout enabled interpreters exists. The `%sparkMahout` and `%flinkMahout` interpreters do not exist by default but can be easily created using this script.
|
||||
|
||||
### Easy Installation
|
||||
To quickly and easily get up and running using Apache Mahout, run the following command from the top-level directory of the Zeppelin install:
|
||||
```bash
|
||||
python scripts/mahout/add_mahout.py
|
||||
```
|
||||
|
||||
This will create the `%sparkMahout` and `%flinkMahout` interpreters, and restart Zeppelin.
|
||||
|
||||
### Advanced Installation
|
||||
|
||||
The `add_mahout.py` script contains several command line arguments for advanced users.
|
||||
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Argument</th>
|
||||
<th>Description</th>
|
||||
<th>Example</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>--zeppelin_home</td>
|
||||
<td>This is the path to the Zeppelin installation. This flag is not needed if the script is run from the top-level installation directory or from the `zeppelin/scripts/mahout` directory.</td>
|
||||
<td>/path/to/zeppelin</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>--mahout_home</td>
|
||||
<td>If the user has already installed Mahout, this flag can set the path to `MAHOUT_HOME`. If this is set, downloading Mahout will be skipped.</td>
|
||||
<td>/path/to/mahout_home</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>--restart_later</td>
|
||||
<td>Restarting is necessary for updates to take effect. By default the script will restart Zeppelin for you- restart will be skipped if this flag is set.</td>
|
||||
<td>NA</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>--force_download</td>
|
||||
<td>This flag will force the script to re-download the binary even if it already exists. This is useful for previously failed downloads.</td>
|
||||
<td>NA</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>--overwrite_existing</td>
|
||||
<td>This flag will force the script to overwrite existing `%sparkMahout` and `%flinkMahout` interpreters. Useful when you want to just start over.</td>
|
||||
<td>NA</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
__NOTE 1:__ Apache Mahout at this time only supports Spark 1.5 and Spark 1.6 and Scala 2.10. If the user is using another version of Spark (e.g. 2.0), the `%sparkMahout` will likely not work. The `%flinkMahout` interpreter will still work and the user is encouraged to develop with that engine as the code can be ported via copy and paste, as is evidenced by the tutorial notebook.
|
||||
|
||||
__NOTE 2:__ If using Apache Flink in cluster mode, the following libraries will also need to be coppied to `${FLINK_HOME}/lib`
|
||||
- mahout-math-0.12.2.jar
|
||||
- mahout-math-scala_2.10-0.12.2.jar
|
||||
- mahout-flink_2.10-0.12.2.jar
|
||||
- mahout-hdfs-0.12.2.jar
|
||||
- [com.google.guava:guava:14.0.1](http://central.maven.org/maven2/com/google/guava/guava/14.0.1/guava-14.0.1.jar)
|
||||
|
||||
## Overview
|
||||
|
||||
The [Apache Mahout](http://mahout.apache.org/)™ project's goal is to build an environment for quickly creating scalable performant machine learning applications.
|
||||
|
||||
Apache Mahout software provides three major features:
|
||||
|
||||
- A simple and extensible programming environment and framework for building scalable algorithms
|
||||
- A wide variety of premade algorithms for Scala + Apache Spark, H2O, Apache Flink
|
||||
- Samsara, a vector math experimentation environment with R-like syntax which works at scale
|
||||
|
||||
In other words:
|
||||
|
||||
*Apache Mahout provides a unified API for quickly creating machine learning algorithms on a variety of engines.*
|
||||
|
||||
## How to use
|
||||
|
||||
When starting a session with Apache Mahout, depending on which engine you are using (Spark or Flink), a few imports must be made and a _Distributed Context_ must be declared. Copy and paste the following code and run once to get started.
|
||||
|
||||
### Flink
|
||||
|
||||
```scala
|
||||
%flinkMahout
|
||||
|
||||
import org.apache.flink.api.scala._
|
||||
import org.apache.mahout.math.drm._
|
||||
import org.apache.mahout.math.drm.RLikeDrmOps._
|
||||
import org.apache.mahout.flinkbindings._
|
||||
import org.apache.mahout.math._
|
||||
import scalabindings._
|
||||
import RLikeOps._
|
||||
|
||||
implicit val ctx = new FlinkDistributedContext(benv)
|
||||
```
|
||||
|
||||
### Spark
|
||||
```scala
|
||||
%sparkMahout
|
||||
|
||||
import org.apache.mahout.math._
|
||||
import org.apache.mahout.math.scalabindings._
|
||||
import org.apache.mahout.math.drm._
|
||||
import org.apache.mahout.math.scalabindings.RLikeOps._
|
||||
import org.apache.mahout.math.drm.RLikeDrmOps._
|
||||
import org.apache.mahout.sparkbindings._
|
||||
|
||||
implicit val sdc: org.apache.mahout.sparkbindings.SparkDistributedContext = sc2sdc(sc)
|
||||
```
|
||||
|
||||
### Same Code, Different Engines
|
||||
|
||||
After importing and setting up the distributed context, the Mahout R-Like DSL is consistent across engines. The following code will run in both `%flinkMahout` and `%sparkMahout`
|
||||
|
||||
```scala
|
||||
val drmData = drmParallelize(dense(
|
||||
(2, 2, 10.5, 10, 29.509541), // Apple Cinnamon Cheerios
|
||||
(1, 2, 12, 12, 18.042851), // Cap'n'Crunch
|
||||
(1, 1, 12, 13, 22.736446), // Cocoa Puffs
|
||||
(2, 1, 11, 13, 32.207582), // Froot Loops
|
||||
(1, 2, 12, 11, 21.871292), // Honey Graham Ohs
|
||||
(2, 1, 16, 8, 36.187559), // Wheaties Honey Gold
|
||||
(6, 2, 17, 1, 50.764999), // Cheerios
|
||||
(3, 2, 13, 7, 40.400208), // Clusters
|
||||
(3, 3, 13, 4, 45.811716)), numPartitions = 2)
|
||||
|
||||
drmData.collect(::, 0 until 4)
|
||||
|
||||
val drmX = drmData(::, 0 until 4)
|
||||
val y = drmData.collect(::, 4)
|
||||
val drmXtX = drmX.t %*% drmX
|
||||
val drmXty = drmX.t %*% y
|
||||
|
||||
|
||||
val XtX = drmXtX.collect
|
||||
val Xty = drmXty.collect(::, 0)
|
||||
val beta = solve(XtX, Xty)
|
||||
```
|
||||
|
||||
## Leveraging Resource Pools and R for Visualization
|
||||
|
||||
Resource Pools are a powerful Zeppelin feature that lets us share information between interpreters. A fun trick is to take the output of our work in Mahout and analyze it in other languages.
|
||||
|
||||
### Setting up a Resource Pool in Flink
|
||||
|
||||
In Spark based interpreters resource pools are accessed via the ZeppelinContext API. To put and get things from the resource pool one can be done simple
|
||||
```scala
|
||||
val myVal = 1
|
||||
z.put("foo", myVal)
|
||||
val myFetchedVal = z.get("foo")
|
||||
```
|
||||
|
||||
To add this functionality to a Flink based interpreter we declare the follwoing
|
||||
|
||||
```scala
|
||||
%flinkMahout
|
||||
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext
|
||||
|
||||
val z = InterpreterContext.get().getResourcePool()
|
||||
```
|
||||
|
||||
Now we can access the resource pool in a consistent manner from the `%flinkMahout` interpreter.
|
||||
|
||||
|
||||
### Passing a variable from Mahout to R and Plotting
|
||||
|
||||
In this simple example, we use Mahout (on Flink or Spark, the code is the same) to create a random matrix and then take the Sin of each element. We then randomly sample the matrix and create a tab separated string. Finally we pass that string to R where it is read as a .tsv file, and a DataFrame is created and plotted using native R plotting libraries.
|
||||
|
||||
```scala
|
||||
val mxRnd = Matrices.symmetricUniformView(5000, 2, 1234)
|
||||
val drmRand = drmParallelize(mxRnd)
|
||||
|
||||
|
||||
val drmSin = drmRand.mapBlock() {case (keys, block) =>
|
||||
val blockB = block.like()
|
||||
for (i <- 0 until block.nrow) {
|
||||
blockB(i, 0) = block(i, 0)
|
||||
blockB(i, 1) = Math.sin((block(i, 0) * 8))
|
||||
}
|
||||
keys -> blockB
|
||||
}
|
||||
|
||||
z.put("sinDrm", org.apache.mahout.math.drm.drmSampleToTSV(drmSin, 0.85))
|
||||
```
|
||||
|
||||
And then in an R paragraph...
|
||||
|
||||
```r
|
||||
%spark.r {"imageWidth": "400px"}
|
||||
|
||||
library("ggplot2")
|
||||
|
||||
sinStr = z.get("flinkSinDrm")
|
||||
|
||||
data <- read.table(text= sinStr, sep="\t", header=FALSE)
|
||||
|
||||
plot(data, col="red")
|
||||
```
|
||||
|
|
@ -53,6 +53,12 @@ The following example demonstrates the basic usage of Markdown in a Zeppelin not
|
|||
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />
|
||||
|
||||
## Mathematical expression
|
||||
|
||||
Markdown interpreter leverages %html display system internally. That means you can mix mathematical expressions with markdown syntax. For more information, please see [Mathematical Expression](../displaysystem/basicdisplaysystem.html#mathematical-expressions) section.
|
||||
|
||||
|
||||
|
||||
### Markdown4j Parser
|
||||
|
||||
`markdown4j` parser provides [YUML](http://yuml.me/) and [Websequence](https://www.websequencediagrams.com/) extensions
|
||||
|
|
|
|||
|
|
@ -86,9 +86,26 @@ print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
|
|||
* Code-completion is currently not implemented.
|
||||
|
||||
## Matplotlib integration
|
||||
The python interpreter can display matplotlib graph with the function `z.show()`.
|
||||
You need to have matplotlib module installed and a XServer running to use this functionality!
|
||||
|
||||
The python interpreter can display matplotlib figures inline automatically using the `pyplot` module:
|
||||
|
||||
```python
|
||||
%python
|
||||
import matplotlib.pyplot as plt
|
||||
plt.plot([1, 2, 3])
|
||||
```
|
||||
This is the recommended method for using matplotlib from within a Zeppelin notebook. The output of this command will by default be converted to HTML by implicitly making use of the `%html` magic. Additional configuration can be achieved using the builtin `z.configure_mpl()` method. For example,
|
||||
|
||||
```python
|
||||
z.configure_mpl(width=400, height=300, fmt='svg')
|
||||
plt.plot([1, 2, 3])
|
||||
```
|
||||
|
||||
Will produce a 400x300 image in SVG format, which by default are normally 600x400 and PNG respectively. In the future, another option called `angular` can be used to make it possible to update a plot produced from one paragraph directly from another (the output will be `%angular` instead of `%html`). However, this feature is already available in the `pyspark` interpreter. More details can be found in the included "Zeppelin Tutorial: Python - matplotlib basic" tutorial notebook.
|
||||
|
||||
If Zeppelin cannot find the matplotlib backend files (which should usually be found in `$ZEPPELIN_HOME/interpreter/lib/python`) in your `PYTHONPATH`, then the backend will automatically be set to agg, and the (otherwise deprecated) instructions below can be used for more limited inline plotting.
|
||||
|
||||
If you are unable to load the inline backend, use `z.show(plt)`:
|
||||
```python
|
||||
%python
|
||||
import matplotlib.pyplot as plt
|
||||
|
|
|
|||
|
|
@ -363,6 +363,11 @@ select * from ${table=defaultTableName} where text like '%${search}%'
|
|||
To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html).
|
||||
|
||||
|
||||
## Matplotlib Integration (pyspark)
|
||||
Both the `python` and `pyspark` interpreters have built-in support for inline visualization using `matplotlib`, a popular plotting library for python. More details can be found in the [python interpreter documentation](../interpreter/python.html), since matplotlib support is identical. More advanced interactive plotting can be done with pyspark through utilizing Zeppelin's built-in [Angular Display System](../displaysystem/back-end-angular.html), as shown below:
|
||||
|
||||
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/matplotlibAngularExample.gif" />
|
||||
|
||||
## Interpreter setting option
|
||||
|
||||
You can choose one of `shared`, `scoped` and `isolated` options wheh you configure Spark interpreter. Spark interpreter creates separated Scala compiler per each notebook but share a single SparkContext in `scoped` mode (experimental). It creates separated SparkContext per each notebook in `isolated` mode.
|
||||
|
|
|
|||
|
|
@ -808,6 +808,127 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Update paragraph configuration
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```PUT``` method update paragraph configuration using given id so that user can change paragraph setting such as graph type, show or hide editor/result and paragraph size, etc. You can update certain fields you want, for example you can update <code>colWidth</code> field only by sending request with payload <code>{"colWidth": 12.0}</code>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/paragraph/[paragraphId]/config```</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Success code</td>
|
||||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Bad Request code</td>
|
||||
<td>400</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Forbidden code</td>
|
||||
<td>403</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Not Found code</td>
|
||||
<td>404</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Fail code</td>
|
||||
<td>500</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>sample JSON input</td>
|
||||
<td><pre>
|
||||
{
|
||||
"colWidth": 6.0,
|
||||
"graph": {
|
||||
"mode": "lineChart",
|
||||
"height": 200.0,
|
||||
"optionOpen": false,
|
||||
"keys": [
|
||||
{
|
||||
"name": "age",
|
||||
"index": 0.0,
|
||||
"aggr": "sum"
|
||||
}
|
||||
],
|
||||
"values": [
|
||||
{
|
||||
"name": "value",
|
||||
"index": 1.0,
|
||||
"aggr": "sum"
|
||||
}
|
||||
],
|
||||
"groups": [],
|
||||
"scatter": {}
|
||||
},
|
||||
"editorHide": true,
|
||||
"editorMode": "ace/mode/markdown",
|
||||
"tableHide": false
|
||||
}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>sample JSON response</td>
|
||||
<td><pre>
|
||||
{
|
||||
"status":"OK",
|
||||
"message":"",
|
||||
"body":{
|
||||
"text":"%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
|
||||
"config":{
|
||||
"colWidth":6.0,
|
||||
"graph":{
|
||||
"mode":"lineChart",
|
||||
"height":200.0,
|
||||
"optionOpen":false,
|
||||
"keys":[
|
||||
{
|
||||
"name":"age",
|
||||
"index":0.0,
|
||||
"aggr":"sum"
|
||||
}
|
||||
],
|
||||
"values":[
|
||||
{
|
||||
"name":"value",
|
||||
"index":1.0,
|
||||
"aggr":"sum"
|
||||
}
|
||||
],
|
||||
"groups":[],
|
||||
"scatter":{}
|
||||
},
|
||||
"tableHide":false,
|
||||
"editorMode":"ace/mode/markdown",
|
||||
"editorHide":true
|
||||
},
|
||||
"settings":{
|
||||
"params":{},
|
||||
"forms":{}
|
||||
},
|
||||
"apps":[],
|
||||
"jobName":"paragraph_1423500782552_-1439281894",
|
||||
"id":"20150210-015302_1492795503",
|
||||
"result":{
|
||||
"code":"SUCCESS",
|
||||
"type":"TABLE",
|
||||
"msg":"age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
|
||||
},
|
||||
"dateCreated":"Feb 10, 2015 1:53:02 AM",
|
||||
"dateStarted":"Jul 3, 2015 1:43:17 PM",
|
||||
"dateFinished":"Jul 3, 2015 1:43:23 PM",
|
||||
"status":"FINISHED",
|
||||
"progressUpdateIntervalMs":500
|
||||
}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br/>
|
||||
### Move a paragraph to the specific index
|
||||
<table class="table-configuration">
|
||||
|
|
@ -835,7 +956,6 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
### Delete a paragraph
|
||||
<table class="table-configuration">
|
||||
|
|
@ -934,7 +1054,8 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
<td> Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<td>sample JSON input</td>
|
||||
<tr>
|
||||
<td>sample JSON input</td>
|
||||
<td><pre>
|
||||
{
|
||||
"paragraphs": [
|
||||
|
|
@ -961,6 +1082,7 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
"config": {},
|
||||
"info": {}
|
||||
}</pre></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>sample JSON response</td>
|
||||
<td><pre>
|
||||
|
|
@ -970,7 +1092,6 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
|
|||
"body": "2AZPHY918"
|
||||
}</pre></td>
|
||||
</tr>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br />
|
||||
|
|
|
|||
|
|
@ -147,6 +147,19 @@ ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
|
|||
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
|
||||
```
|
||||
|
||||
### PAM
|
||||
[PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication
|
||||
moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can
|
||||
either reuse one of these services or create your own for Zeppelin. Activiting PAM authentication requires two parameters:
|
||||
1. realm: The Shiro realm being used
|
||||
2. service: The service configured under `/etc/pam.d/` to be used. The name here needs to be the same as the file name under `/etc/pam.d/`
|
||||
|
||||
```
|
||||
[main]
|
||||
pamRealm=org.apache.zeppelin.realm.PamRealm
|
||||
pamRealm.service=sshd
|
||||
```
|
||||
|
||||
### ZeppelinHub
|
||||
[ZeppelinHub](https://www.zeppelinhub.com) is a service that synchronize your Apache Zeppelin notebooks and enables you to collaborate easily.
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,12 @@ import java.net.URL;
|
|||
import java.net.URLClassLoader;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.flink.api.common.JobID;
|
||||
import org.apache.flink.api.scala.FlinkILoop;
|
||||
import org.apache.flink.configuration.Configuration;
|
||||
import org.apache.flink.runtime.akka.AkkaUtils;
|
||||
import org.apache.flink.runtime.instance.ActorGateway;
|
||||
import org.apache.flink.runtime.messages.JobManagerMessages;
|
||||
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
|
||||
import org.apache.flink.runtime.util.EnvironmentInformation;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
|
|
@ -47,10 +51,12 @@ import scala.Option;
|
|||
import scala.Some;
|
||||
import scala.collection.JavaConversions;
|
||||
import scala.collection.immutable.Nil;
|
||||
import scala.concurrent.duration.FiniteDuration;
|
||||
import scala.runtime.AbstractFunction0;
|
||||
import scala.tools.nsc.Settings;
|
||||
import scala.tools.nsc.interpreter.IMain;
|
||||
import scala.tools.nsc.interpreter.Results;
|
||||
import scala.tools.nsc.settings.MutableSettings;
|
||||
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
|
||||
import scala.tools.nsc.settings.MutableSettings.PathSetting;
|
||||
|
||||
|
|
@ -180,7 +186,13 @@ public class FlinkInterpreter extends Interpreter {
|
|||
BooleanSetting b = (BooleanSetting) settings.usejavacp();
|
||||
b.v_$eq(true);
|
||||
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
|
||||
|
||||
|
||||
// To prevent 'File name too long' error on some file system.
|
||||
MutableSettings.IntSetting numClassFileSetting = settings.maxClassfileName();
|
||||
numClassFileSetting.v_$eq(128);
|
||||
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$maxClassfileName_$eq(
|
||||
numClassFileSetting);
|
||||
|
||||
return settings;
|
||||
}
|
||||
|
||||
|
|
@ -334,6 +346,20 @@ public class FlinkInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
if (localMode()) {
|
||||
// In localMode we can cancel all running jobs,
|
||||
// because the local cluster can only run one job at the time.
|
||||
for (JobID job : this.localFlinkCluster.getCurrentlyRunningJobsJava()) {
|
||||
logger.info("Stop job: " + job);
|
||||
cancelJobLocalMode(job);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cancelJobLocalMode(JobID jobID){
|
||||
FiniteDuration timeout = AkkaUtils.getTimeout(this.localFlinkCluster.configuration());
|
||||
ActorGateway leader = this.localFlinkCluster.getLeaderGateway(timeout);
|
||||
leader.ask(new JobManagerMessages.CancelJob(jobID), timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
312
interpreter/lib/python/backend_zinline.py
Normal file
312
interpreter/lib/python/backend_zinline.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This file provides a static (non-interactive) matplotlib plotting backend
|
||||
# for zeppelin notebooks for use with the python/pyspark interpreters
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import uuid
|
||||
import warnings
|
||||
import base64
|
||||
from io import BytesIO
|
||||
try:
|
||||
from StringIO import StringIO
|
||||
except ImportError:
|
||||
from io import StringIO
|
||||
|
||||
import mpl_config
|
||||
import matplotlib
|
||||
from matplotlib._pylab_helpers import Gcf
|
||||
from matplotlib.backends.backend_agg import new_figure_manager, FigureCanvasAgg
|
||||
from matplotlib.backend_bases import ShowBase, FigureManagerBase
|
||||
from matplotlib.figure import Figure
|
||||
|
||||
########################################################################
|
||||
#
|
||||
# The following functions and classes are for pylab and implement
|
||||
# window/figure managers, etc...
|
||||
#
|
||||
########################################################################
|
||||
|
||||
class Show(ShowBase):
|
||||
"""
|
||||
A callable object that displays the figures to the screen. Valid kwargs
|
||||
include figure width and height (in units supported by the div tag), block
|
||||
(allows users to override blocking behavior regardless of whether or not
|
||||
interactive mode is enabled, currently unused) and close (Implicitly call
|
||||
matplotlib.pyplot.close('all') with each call to show()).
|
||||
"""
|
||||
def __call__(self, close=None, block=None, **kwargs):
|
||||
if close is None:
|
||||
close = mpl_config.get('close')
|
||||
try:
|
||||
managers = Gcf.get_all_fig_managers()
|
||||
if not managers:
|
||||
return
|
||||
|
||||
# Tell zeppelin that the output will be html using the %html magic
|
||||
# We want to do this only once to avoid seeing "%html" printed
|
||||
# directly to the outout when multiple figures are displayed from
|
||||
# one paragraph.
|
||||
if mpl_config.get('angular'):
|
||||
print('%angular')
|
||||
else:
|
||||
print('%html')
|
||||
|
||||
# Show all open figures
|
||||
for manager in managers:
|
||||
manager.show(**kwargs)
|
||||
finally:
|
||||
# This closes all the figures if close is set to True.
|
||||
if close and Gcf.get_all_fig_managers():
|
||||
Gcf.destroy_all()
|
||||
|
||||
|
||||
class FigureCanvasZInline(FigureCanvasAgg):
|
||||
"""
|
||||
The canvas the figure renders into. Calls the draw and print fig
|
||||
methods, creates the renderers, etc...
|
||||
"""
|
||||
def get_bytes(self, **kwargs):
|
||||
"""
|
||||
Get the byte representation of the figure.
|
||||
Should only be used with jpg/png formats.
|
||||
"""
|
||||
# Make sure format is correct
|
||||
fmt = kwargs.get('format', mpl_config.get('format'))
|
||||
if fmt == 'svg':
|
||||
raise ValueError("get_bytes() does not support svg, use png or jpg")
|
||||
|
||||
# Express the image as bytes
|
||||
buf = BytesIO()
|
||||
self.print_figure(buf, **kwargs)
|
||||
byte_str = b"data:image/%s;base64," %fmt
|
||||
byte_str += base64.b64encode(buf.getvalue())
|
||||
|
||||
# Python3 forces all strings to default to unicode, but for raster image
|
||||
# formats (eg png, jpg), we want to work with bytes. Thus this step is
|
||||
# needed to ensure compatability for all python versions.
|
||||
byte_str = byte_str.decode('ascii')
|
||||
buf.close()
|
||||
return byte_str
|
||||
|
||||
def get_svg(self, **kwargs):
|
||||
"""
|
||||
Get the svg representation of the figure.
|
||||
Should only be used with svg format.
|
||||
"""
|
||||
# Make sure format is correct
|
||||
fmt = kwargs.get('format', mpl_config.get('format'))
|
||||
if fmt != 'svg':
|
||||
raise ValueError("get_svg() does not support png or jpg, use svg")
|
||||
|
||||
# For SVG the data string has to be unicode, not bytes
|
||||
buf = StringIO()
|
||||
self.print_figure(buf, **kwargs)
|
||||
svg_str = buf.getvalue()
|
||||
buf.close()
|
||||
return svg_str
|
||||
|
||||
def draw_idle(self, *args, **kwargs):
|
||||
"""
|
||||
Called when the figure gets updated (eg through a plotting command).
|
||||
This is overriden to allow open figures to be reshown after they
|
||||
are updated when mpl_config.get('close') is False.
|
||||
"""
|
||||
if not self._is_idle_drawing:
|
||||
with self._idle_draw_cntx():
|
||||
self.draw(*args, **kwargs)
|
||||
draw_if_interactive()
|
||||
|
||||
|
||||
class FigureManagerZInline(FigureManagerBase):
|
||||
"""
|
||||
Wrap everything up into a window for the pylab interface
|
||||
"""
|
||||
def __init__(self, canvas, num):
|
||||
FigureManagerBase.__init__(self, canvas, num)
|
||||
self.fig_id = "figure_{0}".format(uuid.uuid4().hex)
|
||||
self._shown = False
|
||||
|
||||
def angular_bind(self, **kwargs):
|
||||
"""
|
||||
Bind figure data to Zeppelin's Angular Object Registry.
|
||||
If mpl_config("angular") is True and PY4J is supported, this allows
|
||||
for the possibility to interactively update a figure from a separate
|
||||
paragraph without having to display it multiple times.
|
||||
"""
|
||||
# This doesn't work for SVG so make sure it's not our format
|
||||
fmt = kwargs.get('format', mpl_config.get('format'))
|
||||
if fmt == 'svg':
|
||||
return
|
||||
|
||||
# Get the figure data as a byte array
|
||||
src = self.canvas.get_bytes(**kwargs)
|
||||
|
||||
# Flag to determine whether or not to use
|
||||
# zeppelin's angular display system
|
||||
angular = mpl_config.get('angular')
|
||||
|
||||
# ZeppelinContext instance (requires PY4J)
|
||||
context = mpl_config.get('context')
|
||||
|
||||
# Finally we must ensure that automatic closing is set to False,
|
||||
# as otherwise using the angular display system is pointless
|
||||
close = mpl_config.get('close')
|
||||
|
||||
# If above conditions are met, bind the figure data to
|
||||
# the Angular Object Registry.
|
||||
if not close and angular:
|
||||
if hasattr(context, 'angularBind'):
|
||||
# Binding is performed through figure ID to ensure this works
|
||||
# if multiple figures are open
|
||||
context.angularBind(self.fig_id, src)
|
||||
|
||||
# Zeppelin will automatically replace this value even if it
|
||||
# is updated from another pargraph thanks to the {{}} notation
|
||||
src = "{{%s}}" %self.fig_id
|
||||
else:
|
||||
warnings.warn("Cannot bind figure to Angular Object Registry. "
|
||||
"Check if PY4J is installed.")
|
||||
return src
|
||||
|
||||
def angular_unbind(self):
|
||||
"""
|
||||
Unbind figure from angular display system.
|
||||
"""
|
||||
context = mpl_config.get('context')
|
||||
if hasattr(context, 'angularUnbind'):
|
||||
context.angularUnbind(self.fig_id)
|
||||
|
||||
def destroy(self):
|
||||
"""
|
||||
Called when close=True or implicitly by pyplot.close().
|
||||
Overriden to automatically clean up the angular object registry.
|
||||
"""
|
||||
self.angular_unbind()
|
||||
|
||||
def show(self, **kwargs):
|
||||
if not self._shown:
|
||||
zdisplay(self.canvas.figure, **kwargs)
|
||||
else:
|
||||
self.canvas.draw_idle()
|
||||
self.angular_bind(**kwargs)
|
||||
|
||||
self._shown = True
|
||||
|
||||
|
||||
def draw_if_interactive():
|
||||
"""
|
||||
If interactive mode is on, this allows for updating properties of
|
||||
the figure when each new plotting command is called.
|
||||
"""
|
||||
manager = Gcf.get_active()
|
||||
interactive = matplotlib.is_interactive()
|
||||
angular = mpl_config.get('angular')
|
||||
|
||||
# Don't bother continuing if we aren't in interactive mode
|
||||
# or if there are no active figures. Also pointless to continue
|
||||
# in angular mode as we don't want to reshow the figure.
|
||||
if not interactive or angular or manager is None:
|
||||
return
|
||||
|
||||
# Allow for figure to be reshown if close is false since
|
||||
# this function call implies that it has been updated
|
||||
if not mpl_config.get('close'):
|
||||
manager._shown = False
|
||||
|
||||
|
||||
def new_figure_manager(num, *args, **kwargs):
|
||||
"""
|
||||
Create a new figure manager instance
|
||||
"""
|
||||
# if a main-level app must be created, this (and
|
||||
# new_figure_manager_given_figure) is the usual place to
|
||||
# do it -- see backend_wx, backend_wxagg and backend_tkagg for
|
||||
# examples. Not all GUIs require explicit instantiation of a
|
||||
# main-level app (egg backend_gtk, backend_gtkagg) for pylab
|
||||
FigureClass = kwargs.pop('FigureClass', Figure)
|
||||
thisFig = FigureClass(*args, **kwargs)
|
||||
return new_figure_manager_given_figure(num, thisFig)
|
||||
|
||||
|
||||
def new_figure_manager_given_figure(num, figure):
|
||||
"""
|
||||
Create a new figure manager instance for the given figure.
|
||||
"""
|
||||
canvas = FigureCanvasZInline(figure)
|
||||
manager = FigureManagerZInline(canvas, num)
|
||||
return manager
|
||||
|
||||
|
||||
########################################################################
|
||||
#
|
||||
# Backend specific functions
|
||||
#
|
||||
########################################################################
|
||||
|
||||
def zdisplay(fig, **kwargs):
|
||||
"""
|
||||
Publishes a matplotlib figure to the notebook paragraph output.
|
||||
"""
|
||||
# kwargs can be width or height (in units supported by div tag)
|
||||
width = kwargs.pop('width', 'auto')
|
||||
height = kwargs.pop('height', 'auto')
|
||||
fmt = kwargs.get('format', mpl_config.get('format'))
|
||||
|
||||
# Check if format is supported
|
||||
supported_formats = mpl_config.get('supported_formats')
|
||||
if fmt not in supported_formats:
|
||||
raise ValueError("Unsupported format %s" %fmt)
|
||||
|
||||
# For SVG the data string has to be unicode, not bytes
|
||||
if fmt == 'svg':
|
||||
img = fig.canvas.get_svg(**kwargs)
|
||||
|
||||
# This is needed to ensure the SVG image is the correct size.
|
||||
# We should find a better way to do this...
|
||||
width = '{}px'.format(mpl_config.get('width'))
|
||||
height = '{}px'.format(mpl_config.get('height'))
|
||||
else:
|
||||
# Express the image as bytes
|
||||
src = fig.canvas.manager.angular_bind(**kwargs)
|
||||
img = "<img src={src} style='width={width};height:{height}'>"
|
||||
img = img.format(src=src, width=width, height=height)
|
||||
|
||||
# Print the image to the notebook paragraph via the %html magic
|
||||
html = "<div style='width:{width};height:{height}'>{img}<div>"
|
||||
print(html.format(width=width, height=height, img=img))
|
||||
|
||||
def displayhook():
|
||||
"""
|
||||
Called post paragraph execution if interactive mode is on
|
||||
"""
|
||||
if matplotlib.is_interactive():
|
||||
show()
|
||||
|
||||
########################################################################
|
||||
#
|
||||
# Now just provide the standard names that backend.__init__ is expecting
|
||||
#
|
||||
########################################################################
|
||||
|
||||
# Create a reference to the show function we are using. This is what actually
|
||||
# gets called by matplotlib.pyplot.show().
|
||||
show = Show()
|
||||
|
||||
# Default FigureCanvas and FigureManager classes to use from the backend
|
||||
FigureCanvas = FigureCanvasZInline
|
||||
FigureManager = FigureManagerZInline
|
||||
95
interpreter/lib/python/mpl_config.py
Normal file
95
interpreter/lib/python/mpl_config.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This module provides utitlites for users to configure the inline plotting
|
||||
# backend through a PyZeppelinContext instance (eg, through z.configure_mpl())
|
||||
|
||||
import matplotlib
|
||||
|
||||
def configure(**kwargs):
|
||||
"""
|
||||
Generic configure function.
|
||||
Usage: configure(prop1='foo', prop2='bar', ...)
|
||||
Currently supported zeppelin-specific properties are:
|
||||
interactive - If true show all figures without explicit call to show()
|
||||
via a post-execute hook.
|
||||
angular - If true, bind figures to angular display system.
|
||||
close - If true, close all figures once shown.
|
||||
width, height - Default width / height of the figure in pixels.
|
||||
fontsize - Font size.
|
||||
dpi - dpi of the figure.
|
||||
fmt - Figure format
|
||||
supported_formats - Supported Figure formats ()
|
||||
context - ZeppelinContext instance (requires PY4J)
|
||||
"""
|
||||
_config.update(**kwargs)
|
||||
|
||||
# Broadcast relevant changes to matplotlib RC
|
||||
_on_config_change()
|
||||
|
||||
|
||||
def get(key):
|
||||
"""
|
||||
Get the configuration info given a key
|
||||
"""
|
||||
return _config[key]
|
||||
|
||||
|
||||
def _on_config_change():
|
||||
# dpi
|
||||
dpi = _config['dpi']
|
||||
matplotlib.rcParams['savefig.dpi'] = dpi
|
||||
matplotlib.rcParams['figure.dpi'] = dpi
|
||||
|
||||
# Width and height
|
||||
width = float(_config['width']) / dpi
|
||||
height = float(_config['height']) / dpi
|
||||
matplotlib.rcParams['figure.figsize'] = (width, height)
|
||||
|
||||
# Font size
|
||||
fontsize = _config['fontsize']
|
||||
matplotlib.rcParams['font.size'] = fontsize
|
||||
|
||||
# Default Figure Format
|
||||
fmt = _config['format']
|
||||
supported_formats = _config['supported_formats']
|
||||
if fmt not in supported_formats:
|
||||
raise ValueError("Unsupported format %s" %fmt)
|
||||
matplotlib.rcParams['savefig.format'] = fmt
|
||||
|
||||
# Interactive mode
|
||||
interactive = _config['interactive']
|
||||
matplotlib.interactive(interactive)
|
||||
|
||||
|
||||
def _init_config():
|
||||
dpi = matplotlib.rcParams['savefig.dpi']
|
||||
fmt = matplotlib.rcParams['savefig.format']
|
||||
width, height = matplotlib.rcParams['figure.figsize']
|
||||
fontsize = matplotlib.rcParams['font.size']
|
||||
_config['dpi'] = dpi
|
||||
_config['format'] = fmt
|
||||
_config['width'] = width*dpi
|
||||
_config['height'] = height*dpi
|
||||
_config['fontsize'] = fontsize
|
||||
_config['close'] = True
|
||||
_config['interactive'] = matplotlib.is_interactive()
|
||||
_config['angular'] = False
|
||||
_config['supported_formats'] = ['png', 'jpg', 'svg']
|
||||
_config['context'] = None
|
||||
|
||||
|
||||
_config = {}
|
||||
_init_config()
|
||||
|
|
@ -93,6 +93,15 @@
|
|||
"description": "Whether display app info"
|
||||
}
|
||||
},
|
||||
"option": {
|
||||
"remote": true,
|
||||
"port": -1,
|
||||
"perNote": "shared",
|
||||
"perUser": "scoped",
|
||||
"isExistingProcess": false,
|
||||
"setPermission": false,
|
||||
"users": []
|
||||
},
|
||||
"editor": {
|
||||
"language": "scala",
|
||||
"editOnDblClick": false
|
||||
|
|
@ -115,6 +124,15 @@
|
|||
"description": "Execute multiple SQL concurrently if set true."
|
||||
}
|
||||
},
|
||||
"option": {
|
||||
"remote": true,
|
||||
"port": -1,
|
||||
"perNote": "shared",
|
||||
"perUser": "scoped",
|
||||
"isExistingProcess": false,
|
||||
"setPermission": false,
|
||||
"users": []
|
||||
},
|
||||
"editor": {
|
||||
"language": "sql",
|
||||
"editOnDblClick": false
|
||||
|
|
@ -126,6 +144,15 @@
|
|||
"className": "org.apache.zeppelin.livy.LivyPySparkInterpreter",
|
||||
"properties": {
|
||||
},
|
||||
"option": {
|
||||
"remote": true,
|
||||
"port": -1,
|
||||
"perNote": "shared",
|
||||
"perUser": "scoped",
|
||||
"isExistingProcess": false,
|
||||
"setPermission": false,
|
||||
"users": []
|
||||
},
|
||||
"editor": {
|
||||
"language": "python",
|
||||
"editOnDblClick": false
|
||||
|
|
@ -137,6 +164,15 @@
|
|||
"className": "org.apache.zeppelin.livy.LivySparkRInterpreter",
|
||||
"properties": {
|
||||
},
|
||||
"option": {
|
||||
"remote": true,
|
||||
"port": -1,
|
||||
"perNote": "shared",
|
||||
"perUser": "scoped",
|
||||
"isExistingProcess": false,
|
||||
"setPermission": false,
|
||||
"users": []
|
||||
},
|
||||
"editor": {
|
||||
"language": "r",
|
||||
"editOnDblClick": false
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
772
notebook/2BYEZ5EVK/note.json
Normal file
772
notebook/2BYEZ5EVK/note.json
Normal file
File diff suppressed because one or more lines are too long
696
notebook/2C2AUG798/note.json
Normal file
696
notebook/2C2AUG798/note.json
Normal file
File diff suppressed because one or more lines are too long
3
pom.xml
3
pom.xml
|
|
@ -460,6 +460,9 @@
|
|||
<fileset>
|
||||
<directory>interpreter</directory>
|
||||
<followSymlinks>false</followSymlinks>
|
||||
<excludes>
|
||||
<exclude>lib/**</exclude>
|
||||
</excludes>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -47,6 +47,6 @@ mvn -Dpython.test.exclude='' test -pl python -am
|
|||
|
||||
* JavaBuilder can't send SIGINT signal to interrupt paragraph execution. Therefore interpreter directly send a `kill SIGINT PID` to python process to interrupt execution. Python process catch SIGINT signal with some code defined in bootstrap.py
|
||||
|
||||
* Matplotlib display feature is made with SVG export (in string) and then displays it with html code.
|
||||
* Matplotlib figures are displayed inline with the notebook automatically using a built-in backend for zeppelin in conjunction with a post-execute hook.
|
||||
|
||||
* `%python.sql` support for Pandas DataFrames is optional and provided using https://github.com/yhat/pandasql if user have one installed
|
||||
* `%python.sql` support for Pandas DataFrames is optional and provided using https://github.com/yhat/pandasql if user have one installed
|
||||
|
|
|
|||
|
|
@ -36,7 +36,8 @@
|
|||
<py4j.version>0.9.2</py4j.version>
|
||||
<python.test.exclude>
|
||||
**/PythonInterpreterWithPythonInstalledTest.java,
|
||||
**/PythonInterpreterPandasSqlTest.java
|
||||
**/PythonInterpreterPandasSqlTest.java,
|
||||
**/PythonInterpreterMatplotlibTest.java
|
||||
</python.test.exclude>
|
||||
</properties>
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ import org.apache.zeppelin.interpreter.Interpreter;
|
|||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
|
||||
import org.apache.zeppelin.interpreter.InterpreterGroup;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.apache.zeppelin.scheduler.Job;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
|
|
@ -68,6 +70,12 @@ public class PythonInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void open() {
|
||||
// Add matplotlib display hook
|
||||
InterpreterGroup intpGroup = getInterpreterGroup();
|
||||
if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
|
||||
registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
|
||||
}
|
||||
|
||||
LOG.info("Starting Python interpreter ---->");
|
||||
LOG.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
|
||||
|
||||
|
|
|
|||
|
|
@ -16,10 +16,11 @@
|
|||
# PYTHON 2 / 3 compatibility :
|
||||
# bootstrap.py must be runnable with Python 2 or 3
|
||||
|
||||
# Remove interactive mode displayhook
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import base64
|
||||
import warnings
|
||||
from io import BytesIO
|
||||
try:
|
||||
from StringIO import StringIO
|
||||
|
|
@ -117,6 +118,7 @@ class PyZeppelinContext(object):
|
|||
|
||||
def __init__(self):
|
||||
self.max_result = 1000
|
||||
self._displayhook = lambda *args: None
|
||||
|
||||
def input(self, name, defaultValue=""):
|
||||
print(self.errorMsg)
|
||||
|
|
@ -137,11 +139,14 @@ class PyZeppelinContext(object):
|
|||
elif hasattr(p, '__call__'):
|
||||
p() #error reporting
|
||||
|
||||
def show_dataframe(self, df, **kwargs):
|
||||
def show_dataframe(self, df, show_index=False, **kwargs):
|
||||
"""Pretty prints DF using Table Display System
|
||||
"""
|
||||
limit = len(df) > self.max_result
|
||||
header_buf = StringIO("")
|
||||
if show_index:
|
||||
idx_name = str(df.index.name) if df.index.name is not None else ""
|
||||
header_buf.write(idx_name + "\t")
|
||||
header_buf.write(str(df.columns[0]))
|
||||
for col in df.columns[1:]:
|
||||
header_buf.write("\t")
|
||||
|
|
@ -150,7 +155,11 @@ class PyZeppelinContext(object):
|
|||
|
||||
body_buf = StringIO("")
|
||||
rows = df.head(self.max_result).values if limit else df.values
|
||||
for row in rows:
|
||||
index = df.index.values
|
||||
for idx, row in zip(index, rows):
|
||||
if show_index:
|
||||
body_buf.write("%html <strong>{}</strong>".format(idx))
|
||||
body_buf.write("\t")
|
||||
body_buf.write(str(row[0]))
|
||||
for cell in row[1:]:
|
||||
body_buf.write("\t")
|
||||
|
|
@ -164,7 +173,7 @@ class PyZeppelinContext(object):
|
|||
#)
|
||||
body_buf.close(); header_buf.close()
|
||||
|
||||
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
|
||||
def show_matplotlib(self, p, fmt="png", width="auto", height="auto",
|
||||
**kwargs):
|
||||
"""Matplotlib show function
|
||||
"""
|
||||
|
|
@ -187,6 +196,39 @@ class PyZeppelinContext(object):
|
|||
html = "%html <div style='width:{width};height:{height}'>{img}<div>"
|
||||
print(html.format(width=width, height=height, img=img_str))
|
||||
img.close()
|
||||
|
||||
def configure_mpl(self, **kwargs):
|
||||
import mpl_config
|
||||
mpl_config.configure(**kwargs)
|
||||
|
||||
def _setup_matplotlib(self):
|
||||
# If we don't have matplotlib installed don't bother continuing
|
||||
try:
|
||||
import matplotlib
|
||||
except ImportError:
|
||||
pass
|
||||
# Make sure custom backends are available in the PYTHONPATH
|
||||
rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
|
||||
mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
|
||||
if mpl_path not in sys.path:
|
||||
sys.path.append(mpl_path)
|
||||
|
||||
# Finally check if backend exists, and if so configure as appropriate
|
||||
try:
|
||||
matplotlib.use('module://backend_zinline')
|
||||
import backend_zinline
|
||||
|
||||
# Everything looks good so make config assuming that we are using
|
||||
# an inline backend
|
||||
self._displayhook = backend_zinline.displayhook
|
||||
self.configure_mpl(width=600, height=400, dpi=72,
|
||||
fontsize=10, interactive=True, format='png')
|
||||
except ImportError:
|
||||
# Fall back to Agg if no custom backend installed
|
||||
matplotlib.use('Agg')
|
||||
warnings.warn("Unable to load inline matplotlib backend, "
|
||||
"falling back to Agg")
|
||||
|
||||
|
||||
z = PyZeppelinContext()
|
||||
z._setup_matplotlib()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,169 @@
|
|||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.python;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
|
||||
import org.apache.zeppelin.interpreter.InterpreterGroup;
|
||||
import org.apache.zeppelin.interpreter.InterpreterOutput;
|
||||
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
|
||||
import org.apache.zeppelin.user.AuthenticationInfo;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* In order for this test to work, test env must have installed:
|
||||
* <ol>
|
||||
* - <li>Python</li>
|
||||
* - <li>Matplotlib</li>
|
||||
* <ol>
|
||||
*
|
||||
* Your PYTHONPATH should also contain the directory of the Matplotlib
|
||||
* backend files. Usually these can be found in $ZEPPELIN_HOME/interpreter/lib/python.
|
||||
*
|
||||
* To run manually on such environment, use:
|
||||
* <code>
|
||||
* mvn -Dpython.test.exclude='' test -pl python -am
|
||||
* </code>
|
||||
*/
|
||||
public class PythonInterpreterMatplotlibTest {
|
||||
|
||||
private InterpreterGroup intpGroup;
|
||||
private PythonInterpreter python;
|
||||
|
||||
private InterpreterContext context;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
Properties p = new Properties();
|
||||
p.setProperty("zeppelin.python", "python");
|
||||
p.setProperty("zeppelin.python.maxResult", "100");
|
||||
|
||||
intpGroup = new InterpreterGroup();
|
||||
|
||||
python = new PythonInterpreter(p);
|
||||
python.setInterpreterGroup(intpGroup);
|
||||
python.open();
|
||||
|
||||
List<Interpreter> interpreters = new LinkedList<>();
|
||||
interpreters.add(python);
|
||||
intpGroup.put("note", interpreters);
|
||||
|
||||
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
|
||||
new HashMap<String, Object>(), new GUI(),
|
||||
new AngularObjectRegistry(intpGroup.getId(), null), null,
|
||||
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(
|
||||
new InterpreterOutputListener() {
|
||||
@Override public void onAppend(InterpreterOutput out, byte[] line) {}
|
||||
@Override public void onUpdate(InterpreterOutput out, byte[] output) {}
|
||||
}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void dependenciesAreInstalled() {
|
||||
// matplotlib
|
||||
InterpreterResult ret = python.interpret("import matplotlib", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
|
||||
// inline backend
|
||||
ret = python.interpret("import backend_zinline", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void showPlot() {
|
||||
// Simple plot test
|
||||
InterpreterResult ret;
|
||||
ret = python.interpret("import matplotlib.pyplot as plt", context);
|
||||
ret = python.interpret("z.configure_mpl(interactive=False)", context);
|
||||
ret = python.interpret("plt.plot([1, 2, 3])", context);
|
||||
ret = python.interpret("plt.show()", context);
|
||||
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertEquals(ret.message(), Type.HTML, ret.type());
|
||||
assertTrue(ret.message().contains("data:image/png;base64"));
|
||||
assertTrue(ret.message().contains("<div>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
// Test for when configuration is set to auto-close figures after show().
|
||||
public void testClose() {
|
||||
InterpreterResult ret;
|
||||
InterpreterResult ret1;
|
||||
InterpreterResult ret2;
|
||||
ret = python.interpret("import matplotlib.pyplot as plt", context);
|
||||
ret = python.interpret("z.configure_mpl(interactive=False)", context);
|
||||
ret = python.interpret("plt.plot([1, 2, 3])", context);
|
||||
ret1 = python.interpret("plt.show()", context);
|
||||
|
||||
// Second call to show() should print nothing, and Type should be TEXT.
|
||||
// This is because when close=True, there should be no living instances
|
||||
// of FigureManager, causing show() to return before setting the output
|
||||
// type to HTML.
|
||||
ret = python.interpret("plt.show()", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertEquals(ret.message(), Type.TEXT, ret.type());
|
||||
assertTrue(ret.message().equals(""));
|
||||
|
||||
// Now test that new plot is drawn. It should be identical to the
|
||||
// previous one.
|
||||
ret = python.interpret("plt.plot([1, 2, 3])", context);
|
||||
ret2 = python.interpret("plt.show()", context);
|
||||
assertTrue(ret1.message().equals(ret2.message()));
|
||||
}
|
||||
|
||||
@Test
|
||||
// Test for when configuration is set to not auto-close figures after show().
|
||||
public void testNoClose() {
|
||||
InterpreterResult ret;
|
||||
InterpreterResult ret1;
|
||||
InterpreterResult ret2;
|
||||
ret = python.interpret("import matplotlib.pyplot as plt", context);
|
||||
ret = python.interpret("z.configure_mpl(interactive=False, close=False)", context);
|
||||
ret = python.interpret("plt.plot([1, 2, 3])", context);
|
||||
ret1 = python.interpret("plt.show()", context);
|
||||
|
||||
// Second call to show() should print nothing, and Type should be HTML.
|
||||
// This is because when close=False, there should be living instances
|
||||
// of FigureManager, causing show() to set the output
|
||||
// type to HTML even though the figure is inactive.
|
||||
ret = python.interpret("plt.show()", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertEquals(ret.message(), Type.HTML, ret.type());
|
||||
assertTrue(ret.message().equals(""));
|
||||
|
||||
// Now test that plot can be reshown if it is updated. It should be
|
||||
// different from the previous one because it will plot the same line
|
||||
// again but in a different color.
|
||||
ret = python.interpret("plt.plot([1, 2, 3])", context);
|
||||
ret2 = python.interpret("plt.show()", context);
|
||||
assertTrue(!ret1.message().equals(ret2.message()));
|
||||
}
|
||||
}
|
||||
|
|
@ -159,17 +159,20 @@ public class PythonInterpreterPandasSqlTest {
|
|||
ret = python.interpret("import pandas as pd", context);
|
||||
ret = python.interpret("import numpy as np", context);
|
||||
|
||||
// given a Pandas DataFrame with non-text data
|
||||
// given a Pandas DataFrame with an index and non-text data
|
||||
ret = python.interpret("index = pd.Index([10, 11, 12, 13], name='index_name')", context);
|
||||
ret = python.interpret("d1 = {1 : [np.nan, 1, 2, 3], 'two' : [3., 4., 5., 6.7]}", context);
|
||||
ret = python.interpret("df1 = pd.DataFrame(d1)", context);
|
||||
ret = python.interpret("df1 = pd.DataFrame(d1, index=index)", context);
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
|
||||
// when
|
||||
ret = python.interpret("z.show(df1)", context);
|
||||
ret = python.interpret("z.show(df1, show_index=True)", context);
|
||||
|
||||
// then
|
||||
assertEquals(ret.message(), InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertEquals(ret.message(), Type.TABLE, ret.type());
|
||||
assertTrue(ret.message().indexOf("index_name") == 0);
|
||||
assertTrue(ret.message().indexOf("13") > 0);
|
||||
assertTrue(ret.message().indexOf("nan") > 0);
|
||||
assertTrue(ret.message().indexOf("6.7") > 0);
|
||||
}
|
||||
|
|
|
|||
290
scripts/mahout/add_mahout_interpreters.py
Normal file
290
scripts/mahout/add_mahout_interpreters.py
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
# /**
|
||||
# * Licensed to the Apache Software Foundation (ASF) under one
|
||||
# * or more contributor license agreements. See the NOTICE file
|
||||
# * distributed with this work for additional information
|
||||
# * regarding copyright ownership. The ASF licenses this file
|
||||
# * to you under the Apache License, Version 2.0 (the
|
||||
# * "License"); you may not use this file except in compliance
|
||||
# * with the License. You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.apache.org/licenses/LICENSE-2.0
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# */
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from os.path import isfile
|
||||
from os import getcwd
|
||||
|
||||
from subprocess import call, check_call
|
||||
|
||||
|
||||
#######################################################################################################################
|
||||
# I put these here so it will (hopeully) be easy(er) to bump versions / maintain
|
||||
# If there is demand, we could easily make parts or all comand line arguments as well
|
||||
#######################################################################################################################
|
||||
tar_name = "apache-mahout-distribution-0.12.2.tar.gz"
|
||||
mahout_bin_url = "http://apache.osuosl.org/mahout/0.12.2/%s" % tar_name
|
||||
mahout_version = "0.12.2"
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument("--force_download", help="force download Apache Mahout", action="store_true")
|
||||
parser.add_argument("--restart_later", help="force download Apache Mahout", action="store_true")
|
||||
parser.add_argument("--zeppelin_home", help="path to ZEPPELIN_HOME")
|
||||
parser.add_argument("--mahout_home", help="path to MAHOUT_HOME, use this if you have already installed Apache Mahout")
|
||||
parser.add_argument("--overwrite_existing", help="if %sparkMahout or %flinkMahout exist, delete them and create new ones. Otherwise Fail.", action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
|
||||
class ZeppelinTerpWrangler:
|
||||
def __init__(self, interpreter_json_path):
|
||||
self.interpreter_json_path = interpreter_json_path
|
||||
|
||||
def _getTerpID(self, terpName):
|
||||
terp_id = None
|
||||
for k, v in self.interpreter_json['interpreterSettings'].iteritems():
|
||||
if v['name'] == terpName:
|
||||
terp_id = k
|
||||
break
|
||||
|
||||
return terp_id
|
||||
|
||||
def _terpExists(self, terpName):
|
||||
terp_id = self._getTerpID(terpName)
|
||||
if terp_id == None:
|
||||
return False
|
||||
return True
|
||||
|
||||
def createTerp(self, original_terp_name, new_terp_name, overwrite_existing=True ):
|
||||
|
||||
new_terp_id = new_terp_name
|
||||
if self._terpExists(new_terp_name):
|
||||
print "Found existing '%s' interpreter..." % new_terp_name
|
||||
if overwrite_existing:
|
||||
print "deleting %s from interpreter.json" %new_terp_name
|
||||
del self.interpreter_json['interpreterSettings'][self._getTerpID(new_terp_name)]
|
||||
else:
|
||||
print "exiting program."
|
||||
exit(1)
|
||||
|
||||
orig_terp_id = self._getTerpID(original_terp_name)
|
||||
|
||||
from copy import deepcopy
|
||||
self.interpreter_json['interpreterSettings'][new_terp_id] = deepcopy(
|
||||
self.interpreter_json['interpreterSettings'][orig_terp_id])
|
||||
self.interpreter_json['interpreterSettings'][new_terp_id]['name'] = new_terp_name
|
||||
self.interpreter_json['interpreterSettings'][new_terp_id]['id'] = new_terp_id
|
||||
print "created new interpreter '%s' from interpreter '%s" % (new_terp_name, original_terp_name)
|
||||
|
||||
def _readTerpJson(self):
|
||||
with open(self.interpreter_json_path) as f:
|
||||
self.interpreter_json = json.load(f)
|
||||
|
||||
def _writeTerpJson(self):
|
||||
with open(self.interpreter_json_path, 'wb') as f:
|
||||
json.dump(self.interpreter_json, f, sort_keys=True, indent=4)
|
||||
|
||||
def _updateTerpProp(self, terpName, property, value):
|
||||
terp_id = self._getTerpID(terpName)
|
||||
self.interpreter_json['interpreterSettings'][terp_id]['properties'][property] = value
|
||||
|
||||
def _addTerpDep(self, terpName="", dep="", exclusions=None):
|
||||
if self.interpreter_json == {}:
|
||||
print "no interpreter.json loaded, reading last one downloaded"
|
||||
self._readTerpJson()
|
||||
terp_id = self._getTerpID(terpName)
|
||||
deps = self.interpreter_json['interpreterSettings'][terp_id]['dependencies']
|
||||
|
||||
dep_dict = {
|
||||
u'groupArtifactVersion': dep,
|
||||
u'local': False
|
||||
|
||||
}
|
||||
if exclusions != None:
|
||||
dep_dict["exclusions"] = exclusions
|
||||
deps.append(dep_dict)
|
||||
|
||||
## Remove Duplicate Dependencies
|
||||
seen = set()
|
||||
new_deps = list()
|
||||
for d in deps:
|
||||
t = d.items()
|
||||
if t[0] not in seen:
|
||||
seen.add(t[0])
|
||||
new_deps.append(d)
|
||||
|
||||
self.interpreter_json['interpreterSettings'][terp_id]['dependencies'] = new_deps
|
||||
|
||||
def addMahoutConfig(self, terpName, mahout_home, mahout_version = "0.12.2"):
|
||||
|
||||
print "updating '%s' with Apache Mahout dependencies and settings" % terpName
|
||||
|
||||
terpDeps = ["%s/mahout-math-%s.jar" % (mahout_home, mahout_version),
|
||||
"%s/mahout-math-scala_2.10-%s.jar" % (mahout_home, mahout_version)]
|
||||
|
||||
if "spark" in terpName.lower():
|
||||
configs = {
|
||||
"spark.kryo.referenceTracking": "false",
|
||||
"spark.kryo.registrator": "org.apache.mahout.sparkbindings.io.MahoutKryoRegistrator",
|
||||
"spark.kryoserializer.buffer": "32k",
|
||||
"spark.kryoserializer.buffer.max": "600m",
|
||||
"spark.serializer": "org.apache.spark.serializer.KryoSerializer"
|
||||
}
|
||||
terpDeps.append('%s/mahout-spark_2.10-%s-dependency-reduced.jar' % (mahout_home, mahout_version))
|
||||
terpDeps.append("%s/mahout-spark_2.10-%s.jar" % (mahout_home, mahout_version))
|
||||
terpDeps.append("%s/mahout-spark-shell_2.10-%s.jar" % (mahout_home, mahout_version))
|
||||
|
||||
if "flink" in terpName.lower():
|
||||
configs = {
|
||||
"taskmanager.numberOfTaskSlots" : "12"
|
||||
}
|
||||
addlDeps = [
|
||||
"%s/mahout-flink_2.10-%s.jar" % (mahout_home, mahout_version),
|
||||
"%s/mahout-hdfs-%s.jar" % (mahout_home, mahout_version),
|
||||
"com.google.guava:guava:14.0.1"
|
||||
#"%s/guava-14.0.1.jar" % mahout_home ## reuired in lib dir if running against cluster
|
||||
]
|
||||
for t in addlDeps:
|
||||
terpDeps.append(t)
|
||||
|
||||
for k, v in configs.iteritems():
|
||||
self._updateTerpProp(terpName, k, v)
|
||||
|
||||
for t in terpDeps:
|
||||
self._addTerpDep(terpName, t)
|
||||
|
||||
#######################################################################################################################
|
||||
# Need to be sure we know where Zeppelin Top directory is so we can edit conf files
|
||||
#
|
||||
#######################################################################################################################
|
||||
|
||||
def valid_zeppelin_home(path):
|
||||
return isfile(path + "/bin/zeppelin-daemon.sh")
|
||||
|
||||
if args.zeppelin_home == None:
|
||||
zeppelin_home = getcwd()
|
||||
if (zeppelin_home.split("/")[-1] == "bin") and (isfile("zeppelin-daemon.sh")):
|
||||
print "we're in the zeppelin/bin"
|
||||
zeppelin_home = "/".join(zeppelin_home.split("/")[:-1])
|
||||
print "--zeppelin_home not specified, using %s" % zeppelin_home
|
||||
else:
|
||||
zeppelin_home = args.zeppelin_home
|
||||
|
||||
|
||||
if not valid_zeppelin_home(zeppelin_home):
|
||||
print "%s does not appear to be a valid ZEPPELIN_HOME - e.g. the top level directory of the ZEPPELIN install" % zeppelin_home
|
||||
exit(1)
|
||||
else:
|
||||
print "ZEPPELIN_HOME validated"
|
||||
|
||||
interpreter_json_path = zeppelin_home + "/conf/interpreter.json"
|
||||
|
||||
if not isfile(interpreter_json_path):
|
||||
print "interpreter.json doesn't exist. Checking weather Zeppelin is running."
|
||||
status = call(["bin/zeppelin-daemon.sh", 'status'], cwd=zeppelin_home)
|
||||
if status == 1:
|
||||
print "Zeppelin doesn't appear to be running- it is possible that Zeppelin has never been run (interpreter.json is created when Zeppelin is run)"
|
||||
print "I'm going to try to start Zeppelin to create interpreter.json"
|
||||
call(["bin/zeppelin-daemon.sh", 'start'], cwd=zeppelin_home)
|
||||
from time import sleep
|
||||
sleep(3)
|
||||
else:
|
||||
print "We're in the correct top-level directory, Zeppelin appears to be running, but there is no 'interpreter.json'. \
|
||||
\nThis is a confusing case. Please try restarting Zeppelin, but if that doesn't work reach out on the mailing list."
|
||||
|
||||
if isfile(interpreter_json_path):
|
||||
z = ZeppelinTerpWrangler(interpreter_json_path)
|
||||
else:
|
||||
print "'interpreter.json' not found in %s/conf" % args.zeppelin_home
|
||||
exit(1)
|
||||
|
||||
#######################################################################################################################
|
||||
# If --mahout_home not set, download and untar Mahout in to ZEPPELIN_HOME
|
||||
# Set MAHOUT_HOME to ZEPPELIN_HOME/<mahout_untar_dir>
|
||||
#######################################################################################################################
|
||||
|
||||
def download_mahout():
|
||||
if args.force_download:
|
||||
print "--force_download: OK, deleting existing tar if it exists."
|
||||
call(["rm", "%s/%s" % (zeppelin_home, tar_name)])
|
||||
return True
|
||||
elif isfile("%s/%s" % (zeppelin_home, tar_name)):
|
||||
print "%s found, skipping download" % tar_name
|
||||
return False
|
||||
elif args.mahout_home:
|
||||
print "--mahout_home set, skipping download"
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
if download_mahout():
|
||||
check_call(['wget', mahout_bin_url], cwd= zeppelin_home)
|
||||
check_call(['tar', 'xzf', tar_name], cwd= zeppelin_home)
|
||||
|
||||
|
||||
|
||||
if args.mahout_home:
|
||||
mahout_home = args.mahout_home
|
||||
else:
|
||||
mahout_home = zeppelin_home + "/" + ".".join(tar_name.split(".")[:-2])
|
||||
|
||||
#######################################################################################################################
|
||||
# Create new interpreters
|
||||
#######################################################################################################################
|
||||
|
||||
z._readTerpJson()
|
||||
z.createTerp("spark", "sparkMahout", args.overwrite_existing)
|
||||
z.createTerp("flink", "flinkMahout", args.overwrite_existing)
|
||||
z.addMahoutConfig("sparkMahout", mahout_home, mahout_version)
|
||||
z.addMahoutConfig("flinkMahout", mahout_home, mahout_version)
|
||||
z._writeTerpJson()
|
||||
|
||||
#######################################################################################################################
|
||||
# Add "export MAHOUT_HOME=... to conf/zeppelin-env.sh
|
||||
# Create if doesn't exist.
|
||||
#######################################################################################################################
|
||||
|
||||
mahout_home_str = '\nexport MAHOUT_HOME=%s\n' % (mahout_home)
|
||||
|
||||
zeppelin_env_sh_path = '%s/conf/zeppelin-env.sh' % zeppelin_home
|
||||
if isfile(zeppelin_env_sh_path):
|
||||
with open(zeppelin_env_sh_path, 'rb') as f:
|
||||
zeppelin_env_sh = f.readlines()
|
||||
if any(["export MAHOUT_HOME=" in line for line in zeppelin_env_sh]):
|
||||
print "'export MAHOUT_HOME=...' already exists in zeppelin_env.sh, not appending"
|
||||
else:
|
||||
print "appending '%s' to conf/zeppelin-env.sh" % mahout_home_str
|
||||
with open(zeppelin_env_sh_path, 'a') as f:
|
||||
f.write(mahout_home_str)
|
||||
else:
|
||||
print "appending '%s' to conf/zeppelin-env.sh" % mahout_home_str
|
||||
with open(zeppelin_env_sh_path, 'wb') as f:
|
||||
f.write(mahout_home_str)
|
||||
|
||||
|
||||
#######################################################################################################################
|
||||
# You have to restart Apache Zeppelin for new terps to show up... do this for user unless the specified otherwise
|
||||
#
|
||||
#######################################################################################################################
|
||||
if not args.restart_later:
|
||||
print "restarting Apache Zeppelin to load new interpreters..."
|
||||
check_call(["bin/zeppelin-daemon.sh", 'restart'], cwd= zeppelin_home)
|
||||
else:
|
||||
print "--restart_later flag detected: remember to restart Zeppelin to see new Mahout interpreters!!"
|
||||
|
||||
#######################################################################################################################
|
||||
# Good bye
|
||||
#######################################################################################################################
|
||||
|
||||
print "---------------------------------------------------------------------------------------------------------------"
|
||||
print "all done! Thanks for using Apache Mahout"
|
||||
print "bye"
|
||||
|
|
@ -48,7 +48,9 @@ import org.apache.spark.sql.SQLContext;
|
|||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.apache.zeppelin.interpreter.InterpreterGroup;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
|
||||
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
|
||||
import org.apache.zeppelin.interpreter.WrappedInterpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
|
|
@ -111,6 +113,11 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
@Override
|
||||
public void open() {
|
||||
// Add matplotlib display hook
|
||||
InterpreterGroup intpGroup = getInterpreterGroup();
|
||||
if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
|
||||
registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
|
||||
}
|
||||
DepInterpreter depInterpreter = getDepInterpreter();
|
||||
|
||||
// load libraries from Dependency Interpreter
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
import sys, getopt, traceback, json, re
|
||||
import os, sys, getopt, traceback, json, re
|
||||
|
||||
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
|
||||
from py4j.protocol import Py4JJavaError
|
||||
|
|
@ -50,6 +50,7 @@ class Logger(object):
|
|||
class PyZeppelinContext(dict):
|
||||
def __init__(self, zc):
|
||||
self.z = zc
|
||||
self._displayhook = lambda *args: None
|
||||
|
||||
def show(self, obj):
|
||||
from pyspark.sql import DataFrame
|
||||
|
|
@ -116,6 +117,39 @@ class PyZeppelinContext(dict):
|
|||
return self.z.getHook(event)
|
||||
return self.z.getHook(event, replName)
|
||||
|
||||
def _setup_matplotlib(self):
|
||||
# If we don't have matplotlib installed don't bother continuing
|
||||
try:
|
||||
import matplotlib
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
# Make sure custom backends are available in the PYTHONPATH
|
||||
rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd())
|
||||
mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python')
|
||||
if mpl_path not in sys.path:
|
||||
sys.path.append(mpl_path)
|
||||
|
||||
# Finally check if backend exists, and if so configure as appropriate
|
||||
try:
|
||||
matplotlib.use('module://backend_zinline')
|
||||
import backend_zinline
|
||||
|
||||
# Everything looks good so make config assuming that we are using
|
||||
# an inline backend
|
||||
self._displayhook = backend_zinline.displayhook
|
||||
self.configure_mpl(width=600, height=400, dpi=72, fontsize=10,
|
||||
interactive=True, format='png', context=self.z)
|
||||
except ImportError:
|
||||
# Fall back to Agg if no custom backend installed
|
||||
matplotlib.use('Agg')
|
||||
warnings.warn("Unable to load inline matplotlib backend, "
|
||||
"falling back to Agg")
|
||||
|
||||
def configure_mpl(self, **kwargs):
|
||||
import mpl_config
|
||||
mpl_config.configure(**kwargs)
|
||||
|
||||
def __tupleToScalaTuple2(self, tuple):
|
||||
if (len(tuple) == 2):
|
||||
return gateway.jvm.scala.Tuple2(tuple[0], tuple[1])
|
||||
|
|
@ -244,6 +278,7 @@ sqlContext = sqlc
|
|||
|
||||
completion = PySparkCompletion(intp)
|
||||
z = PyZeppelinContext(intp.getZeppelinContext())
|
||||
z._setup_matplotlib()
|
||||
|
||||
while True :
|
||||
req = intp.getStatements()
|
||||
|
|
@ -251,6 +286,22 @@ while True :
|
|||
stmts = req.statements().split("\n")
|
||||
jobGroup = req.jobGroup()
|
||||
final_code = []
|
||||
|
||||
# Get post-execute hooks
|
||||
try:
|
||||
global_hook = intp.getHook('post_exec_dev')
|
||||
except:
|
||||
global_hook = None
|
||||
|
||||
try:
|
||||
user_hook = z.getHook('post_exec')
|
||||
except:
|
||||
user_hook = None
|
||||
|
||||
nhooks = 0
|
||||
for hook in (global_hook, user_hook):
|
||||
if hook:
|
||||
nhooks += 1
|
||||
|
||||
for s in stmts:
|
||||
if s == None:
|
||||
|
|
@ -268,7 +319,9 @@ while True :
|
|||
# so that the last statement's evaluation will be printed to stdout
|
||||
sc.setJobGroup(jobGroup, "Zeppelin")
|
||||
code = compile('\n'.join(final_code), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
|
||||
to_run_exec, to_run_single = code.body[:-1], code.body[-1:]
|
||||
to_run_hooks = code.body[-nhooks:]
|
||||
to_run_exec, to_run_single = (code.body[:-(nhooks + 1)],
|
||||
[code.body[-(nhooks + 1)]])
|
||||
|
||||
try:
|
||||
for node in to_run_exec:
|
||||
|
|
@ -280,6 +333,11 @@ while True :
|
|||
mod = ast.Interactive([node])
|
||||
code = compile(mod, '<stdin>', 'single')
|
||||
exec(code)
|
||||
|
||||
for node in to_run_hooks:
|
||||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code)
|
||||
except:
|
||||
raise Exception(traceback.format_exc())
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<!-- Enable access to all projects in the current multimodule build!
|
||||
<!-- Enable access to all projects in the current multimodule build!
|
||||
<useAllReactorProjects>true</useAllReactorProjects> -->
|
||||
<!-- Now, select which projects to include in this module-set. -->
|
||||
<includes>
|
||||
|
|
@ -92,4 +92,3 @@
|
|||
</fileSets>-->
|
||||
|
||||
</assembly>
|
||||
|
||||
|
|
|
|||
|
|
@ -163,6 +163,8 @@ The following components are provided under Apache License.
|
|||
(Apache 2.0) tez-runtime-internals (org.apache.tez:tez-runtime-internals:0.7.0 - http://tez.apache.org)
|
||||
(Apache 2.0) tez-mapreduce (org.apache.tez:tez-mapreduce:0.7.0 - http://tez.apache.org)
|
||||
(Apache 2.0) tez-yarn-timeline-history-with-acls (org.apache.tez:tez-yarn-timeline-history-with-acls:0.7.0 - http://tez.apache.org)
|
||||
(Apache 2.0) jna (net.java.dev.jna:jna:4.1.0 https://github.com/java-native-access/jna)
|
||||
(Apache 2.0) MathJax v2.7.0 - https://github.com/mathjax/MathJax/blob/2.7.0/LICENSE
|
||||
|
||||
========================================================================
|
||||
MIT licenses
|
||||
|
|
@ -208,7 +210,8 @@ The following components are provided under the MIT License.
|
|||
(The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.16 - http://www.slf4j.org)
|
||||
(The MIT License) angular-resource (angular-resource - https://github.com/angular/angular.js/tree/master/src/ngResource)
|
||||
(The MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.4 - https://github.com/ralfstx/minimal-json)
|
||||
(The MIT License) pyrolite (net.razorvine:pyrolite:4.9) - https://github.com/irmen/Pyrolite/blob/v4.9/LICENSE
|
||||
(The MIT License) pyrolite (net.razorvine:pyrolite:4.9) - https://github.com/irmen/Pyrolite/blob/v4.9/LICENSE)
|
||||
(The MIT License) libpam4j (org.kohsuke:libpam4j:1.8 https://github.com/kohsuke/libpam4j/blob/master/src/main/java/org/jvnet/libpam/PAM.java)
|
||||
|
||||
========================================================================
|
||||
BSD-style licenses
|
||||
|
|
|
|||
|
|
@ -319,6 +319,7 @@ public abstract class Interpreter {
|
|||
private Map<String, InterpreterProperty> properties;
|
||||
private Map<String, Object> editor;
|
||||
private String path;
|
||||
private InterpreterOption option;
|
||||
|
||||
public RegisteredInterpreter(String name, String group, String className,
|
||||
Map<String, InterpreterProperty> properties) {
|
||||
|
|
@ -376,6 +377,9 @@ public abstract class Interpreter {
|
|||
return getGroup() + "." + getName();
|
||||
}
|
||||
|
||||
public InterpreterOption getOption() {
|
||||
return option;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@
|
|||
|
||||
package org.apache.zeppelin.interpreter;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
|
|
@ -77,9 +75,12 @@ public class InterpreterOption {
|
|||
}
|
||||
|
||||
public InterpreterOption(boolean remote, String perUser, String perNote) {
|
||||
Preconditions.checkNotNull(remote);
|
||||
Preconditions.checkNotNull(perUser);
|
||||
Preconditions.checkNotNull(perNote);
|
||||
if (perUser == null) {
|
||||
throw new NullPointerException("perUser can not be null.");
|
||||
}
|
||||
if (perNote == null) {
|
||||
throw new NullPointerException("perNote can not be null.");
|
||||
}
|
||||
|
||||
this.remote = remote;
|
||||
this.perUser = perUser;
|
||||
|
|
@ -213,11 +213,17 @@ public class InterpreterOutput extends OutputStream {
|
|||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private boolean typeShouldBeDetected() {
|
||||
return getType() == InterpreterResult.Type.TABLE ? false : true;
|
||||
}
|
||||
|
||||
public void flush() throws IOException {
|
||||
synchronized (outList) {
|
||||
buffer.flush();
|
||||
byte[] bytes = buffer.toByteArray();
|
||||
bytes = detectTypeFromLine(bytes);
|
||||
if (typeShouldBeDetected()) {
|
||||
bytes = detectTypeFromLine(bytes);
|
||||
}
|
||||
if (bytes != null) {
|
||||
outList.add(bytes);
|
||||
if (type == InterpreterResult.Type.TEXT) {
|
||||
|
|
|
|||
|
|
@ -124,14 +124,20 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
|
|||
|
||||
long startTime = System.currentTimeMillis();
|
||||
while (System.currentTimeMillis() - startTime < getConnectTimeout()) {
|
||||
if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) {
|
||||
break;
|
||||
} else {
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
|
||||
"Thread.sleep", e);
|
||||
try {
|
||||
if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) {
|
||||
break;
|
||||
} else {
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
logger.error("Exception in RemoteInterpreterProcess while synchronized reference " +
|
||||
"Thread.sleep", e);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Remote interpreter not yet accessible at localhost:" + port);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.ServerSocket;
|
||||
import java.net.Socket;
|
||||
|
|
@ -46,9 +47,19 @@ public class RemoteInterpreterUtils {
|
|||
discover.connect(new InetSocketAddress(host, port), 1000);
|
||||
discover.close();
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
} catch (ConnectException cne) {
|
||||
// end point is not accessible
|
||||
LOGGER.debug(e.getMessage(), e);
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("Remote endpoint '" + host + ":" + port + "' is not accessible " +
|
||||
"(might be initializing): " + cne.getMessage());
|
||||
}
|
||||
return false;
|
||||
} catch (IOException ioe) {
|
||||
// end point is not accessible
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug("Remote endpoint '" + host + ":" + port + "' is not accessible " +
|
||||
"(might be initializing): " + ioe.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,6 +115,13 @@ public class InterpreterOutputTest implements InterpreterOutputListener {
|
|||
assertEquals(InterpreterResult.Type.HTML, out.getType());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMagicData() throws IOException {
|
||||
out.write("%table col1\tcol2\n%html <h3> This is a hack </h3>\t234\n".getBytes());
|
||||
assertEquals(InterpreterResult.Type.TABLE, out.getType());
|
||||
assertEquals("col1\tcol2\n%html <h3> This is a hack </h3>\t234\n", new String(out.toByteArray()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onAppend(InterpreterOutput out, byte[] line) {
|
||||
numAppendEvent++;
|
||||
|
|
|
|||
|
|
@ -319,6 +319,10 @@
|
|||
<groupId>org.eclipse.jetty.websocket</groupId>
|
||||
<artifactId>websocket-client</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
|
@ -353,6 +357,22 @@
|
|||
<groupId>org.apache.shiro</groupId>
|
||||
<artifactId>shiro-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.kohsuke</groupId>
|
||||
<artifactId>libpam4j</artifactId>
|
||||
<version>1.8</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.realm;
|
||||
|
||||
import org.apache.shiro.authc.AuthenticationException;
|
||||
import org.apache.shiro.authc.AuthenticationInfo;
|
||||
import org.apache.shiro.authc.AuthenticationToken;
|
||||
import org.apache.shiro.authc.SimpleAuthenticationInfo;
|
||||
import org.apache.shiro.authc.UsernamePasswordToken;
|
||||
import org.apache.shiro.authz.AuthorizationInfo;
|
||||
import org.apache.shiro.authz.SimpleAuthorizationInfo;
|
||||
import org.apache.shiro.crypto.hash.DefaultHashService;
|
||||
import org.apache.shiro.crypto.hash.Hash;
|
||||
import org.apache.shiro.crypto.hash.HashRequest;
|
||||
import org.apache.shiro.crypto.hash.HashService;
|
||||
import org.apache.shiro.realm.AuthorizingRealm;
|
||||
import org.apache.shiro.subject.PrincipalCollection;
|
||||
import org.jvnet.libpam.PAM;
|
||||
import org.jvnet.libpam.PAMException;
|
||||
import org.jvnet.libpam.UnixUser;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* An {@code AuthorizingRealm} base on libpam4j.
|
||||
*/
|
||||
public class PamRealm extends AuthorizingRealm {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubRealm.class);
|
||||
|
||||
private String service;
|
||||
|
||||
@Override
|
||||
protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
|
||||
Set<String> roles = new LinkedHashSet<>();
|
||||
|
||||
UserPrincipal user = principals.oneByType(UserPrincipal.class);
|
||||
|
||||
if (user != null){
|
||||
roles.addAll(user.getUnixUser().getGroups());
|
||||
}
|
||||
|
||||
return new SimpleAuthorizationInfo(roles);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token)
|
||||
throws AuthenticationException {
|
||||
|
||||
UsernamePasswordToken userToken = (UsernamePasswordToken) token;
|
||||
UnixUser user;
|
||||
|
||||
try {
|
||||
user = (new PAM(this.getService()))
|
||||
.authenticate(userToken.getUsername(), new String(userToken.getPassword()));
|
||||
} catch (PAMException e) {
|
||||
throw new AuthenticationException("Authentication failed for PAM.", e);
|
||||
}
|
||||
|
||||
return new SimpleAuthenticationInfo(
|
||||
new UserPrincipal(user),
|
||||
userToken.getCredentials(),
|
||||
getName());
|
||||
}
|
||||
|
||||
public String getService() {
|
||||
return service;
|
||||
}
|
||||
|
||||
public void setService(String service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.realm;
|
||||
|
||||
import org.jvnet.libpam.UnixUser;
|
||||
|
||||
import java.security.Principal;
|
||||
|
||||
/**
|
||||
* A {@code java.security.Principal} implememtation for use with Shiro {@code PamRealm}
|
||||
*/
|
||||
public class UserPrincipal implements Principal {
|
||||
private final UnixUser userName;
|
||||
|
||||
public UserPrincipal(UnixUser userName) {
|
||||
this.userName = userName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return userName.getUserName();
|
||||
}
|
||||
|
||||
public UnixUser getUnixUser() {
|
||||
return userName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.valueOf(userName.getUserName());
|
||||
}
|
||||
}
|
||||
|
|
@ -42,6 +42,7 @@ import org.apache.zeppelin.notebook.Note;
|
|||
import org.apache.zeppelin.notebook.Notebook;
|
||||
import org.apache.zeppelin.notebook.NotebookAuthorization;
|
||||
import org.apache.zeppelin.notebook.Paragraph;
|
||||
import org.apache.zeppelin.rest.exception.BadRequestException;
|
||||
import org.apache.zeppelin.rest.exception.NotFoundException;
|
||||
import org.apache.zeppelin.rest.exception.ForbiddenException;
|
||||
import org.apache.zeppelin.rest.message.CronRequest;
|
||||
|
|
@ -451,6 +452,38 @@ public class NotebookRestApi {
|
|||
return new JsonResponse<>(Status.OK, "", p).build();
|
||||
}
|
||||
|
||||
@PUT
|
||||
@Path("{noteId}/paragraph/{paragraphId}/config")
|
||||
@ZeppelinApi
|
||||
public Response updateParagraphConfig(@PathParam("noteId") String noteId,
|
||||
@PathParam("paragraphId") String paragraphId, String message) throws IOException {
|
||||
String user = SecurityUtils.getPrincipal();
|
||||
LOG.info("{} will update paragraph config {} {}", user, noteId, paragraphId);
|
||||
|
||||
Note note = notebook.getNote(noteId);
|
||||
checkIfNoteIsNotNull(note);
|
||||
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot update this paragraph config");
|
||||
Paragraph p = note.getParagraph(paragraphId);
|
||||
checkIfParagraphIsNotNull(p);
|
||||
|
||||
Map<String, Object> newConfig = gson.fromJson(message, HashMap.class);
|
||||
if (newConfig == null || newConfig.isEmpty()) {
|
||||
LOG.warn("{} is trying to update paragraph {} of note {} with empty config",
|
||||
user, paragraphId, noteId);
|
||||
throw new BadRequestException("paragraph config cannot be empty");
|
||||
}
|
||||
Map<String, Object> origConfig = p.getConfig();
|
||||
for (String key : newConfig.keySet()) {
|
||||
origConfig.put(key, newConfig.get(key));
|
||||
}
|
||||
|
||||
p.setConfig(origConfig);
|
||||
AuthenticationInfo subject = new AuthenticationInfo(user);
|
||||
note.persist(subject);
|
||||
|
||||
return new JsonResponse<>(Status.OK, "", p).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Move paragraph REST API
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.rest.exception;
|
||||
|
||||
import org.apache.zeppelin.utils.ExceptionUtils;
|
||||
|
||||
import javax.ws.rs.WebApplicationException;
|
||||
import javax.ws.rs.core.Response;
|
||||
|
||||
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
|
||||
|
||||
/**
|
||||
* BadRequestException handler for WebApplicationException.
|
||||
*/
|
||||
public class BadRequestException extends WebApplicationException {
|
||||
|
||||
public BadRequestException() {
|
||||
super(ExceptionUtils.jsonResponse(BAD_REQUEST));
|
||||
}
|
||||
|
||||
private static Response badRequestJson(String message) {
|
||||
return ExceptionUtils.jsonResponseContent(BAD_REQUEST, message);
|
||||
}
|
||||
|
||||
public BadRequestException(Throwable cause, String message) {
|
||||
super(cause, badRequestJson(message));
|
||||
}
|
||||
|
||||
public BadRequestException(String message) {
|
||||
super(badRequestJson(message));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.realm;
|
||||
|
||||
import org.apache.shiro.authc.AuthenticationInfo;
|
||||
import org.apache.shiro.authc.UsernamePasswordToken;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assume.assumeTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* The test will only be executed if the environment variables PAM_USER and PAM_PASS are present. They should
|
||||
* contain username and password of an valid system user to make the test pass. The service needs to be configured
|
||||
* under /etc/pam.d/sshd to resolve and authenticate the system user.
|
||||
*
|
||||
* Contains main() function so the test can be executed manually.
|
||||
*
|
||||
* Set in MacOS to run in IDE(A):
|
||||
* $ launchctl setenv PAM_USER user
|
||||
* $ launchctl setenv PAM_PASS xxxxx
|
||||
*/
|
||||
public class PamRealmTest {
|
||||
|
||||
@Test
|
||||
public void testDoGetAuthenticationInfo() {
|
||||
PamRealm realm = new PamRealm();
|
||||
realm.setService("sshd");
|
||||
|
||||
String pam_user = System.getenv("PAM_USER");
|
||||
String pam_pass = System.getenv("PAM_PASS");
|
||||
assumeTrue(pam_user != null);
|
||||
assumeTrue(pam_pass != null);
|
||||
|
||||
// mock shiro auth token
|
||||
UsernamePasswordToken authToken = mock(UsernamePasswordToken.class);
|
||||
when(authToken.getUsername()).thenReturn(pam_user);
|
||||
when(authToken.getPassword()).thenReturn(pam_pass.toCharArray());
|
||||
when(authToken.getCredentials()).thenReturn(pam_pass);
|
||||
|
||||
AuthenticationInfo authInfo = realm.doGetAuthenticationInfo(authToken);
|
||||
|
||||
assertTrue(authInfo.getCredentials() != null);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
PamRealmTest test = new PamRealmTest();
|
||||
test.testDoGetAuthenticationInfo();
|
||||
}
|
||||
}
|
||||
|
|
@ -94,7 +94,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
|
|||
Note note2 = ZeppelinServer.notebook.createNote(anonymous);
|
||||
// Set only writers
|
||||
jsonRequest = "{\"readers\":[],\"owners\":[]," +
|
||||
"\"writers\":[\"admin-team\"]}";
|
||||
"\"writers\":[\"admin-team\"]}";
|
||||
put = httpPut("/notebook/" + note2.getId() + "/permissions/", jsonRequest);
|
||||
assertThat("test update method:", put, isAllowed());
|
||||
put.releaseConnection();
|
||||
|
|
@ -175,7 +175,32 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
|
|||
//cleanup
|
||||
ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
|
||||
ZeppelinServer.notebook.removeNote(clonedNoteId, anonymous);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdateParagraphConfig() throws IOException {
|
||||
Note note = ZeppelinServer.notebook.createNote(anonymous);
|
||||
String noteId = note.getId();
|
||||
Paragraph p = note.addParagraph();
|
||||
assertNull(p.getConfig().get("colWidth"));
|
||||
String paragraphId = p.getId();
|
||||
String jsonRequest = "{\"colWidth\": 6.0}";
|
||||
|
||||
PutMethod put = httpPut("/notebook/" + noteId + "/paragraph/" + paragraphId +"/config", jsonRequest);
|
||||
assertThat("test testUpdateParagraphConfig:", put, isAllowed());
|
||||
|
||||
Map<String, Object> resp = gson.fromJson(put.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
|
||||
}.getType());
|
||||
Map<String, Object> respBody = (Map<String, Object>) resp.get("body");
|
||||
Map<String, Object> config = (Map<String, Object>) respBody.get("config");
|
||||
put.releaseConnection();
|
||||
|
||||
assertEquals(config.get("colWidth"), 6.0);
|
||||
note = ZeppelinServer.notebook.getNote(noteId);
|
||||
assertEquals(note.getParagraph(paragraphId).getConfig().get("colWidth"), 6.0);
|
||||
|
||||
//cleanup
|
||||
ZeppelinServer.notebook.removeNote(noteId, anonymous);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@
|
|||
"nv": false,
|
||||
"ace": false,
|
||||
"d3": false,
|
||||
"MathJax": false,
|
||||
"BootstrapDialog": false,
|
||||
"Handsontable": false,
|
||||
"moment": false
|
||||
"moment": false,
|
||||
"zeppelin" : false
|
||||
},
|
||||
"rules": {
|
||||
"no-bitwise": 2,
|
||||
|
|
|
|||
|
|
@ -466,6 +466,12 @@ module.exports = function(grunt) {
|
|||
cwd: 'bower_components/jquery-ui/themes/base/images',
|
||||
src: '{,*/}*.{png,jpg,jpeg,gif}',
|
||||
dest: '<%= yeoman.dist %>/styles/images'
|
||||
}, {
|
||||
expand: true,
|
||||
cwd: 'bower_components/MathJax',
|
||||
src: [
|
||||
'extensions/**', 'jax/**', 'fonts/**'],
|
||||
dest: '<%= yeoman.dist %>'
|
||||
}]
|
||||
},
|
||||
styles: {
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@
|
|||
"handsontable": "~0.24.2",
|
||||
"moment-duration-format": "^1.3.0",
|
||||
"select2": "^4.0.3",
|
||||
"github-markdown-css": "^2.4.0"
|
||||
"github-markdown-css": "^2.4.0",
|
||||
"MathJax": "2.7.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"angular-mocks": "1.5.0"
|
||||
|
|
|
|||
|
|
@ -19,39 +19,10 @@ limitations under the License.
|
|||
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('table')}"
|
||||
ng-click="setGraphMode('table', true)"
|
||||
tooltip="Table" tooltip-placement="bottom"><i class="fa fa-table"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('multiBarChart')}"
|
||||
ng-click="setGraphMode('multiBarChart', true)"
|
||||
tooltip="Bar Chart" tooltip-placement="bottom"><i class="fa fa-bar-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('pieChart')}"
|
||||
ng-click="setGraphMode('pieChart', true)"
|
||||
tooltip="Pie Chart" tooltip-placement="bottom"><i class="fa fa-pie-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('stackedAreaChart')}"
|
||||
ng-click="setGraphMode('stackedAreaChart', true)"
|
||||
tooltip="Area Chart" tooltip-placement="bottom"><i class="fa fa-area-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('lineChart') || isGraphMode('lineWithFocusChart')}"
|
||||
ng-click="paragraph.config.graph.lineWithFocus ? setGraphMode('lineWithFocusChart', true) : setGraphMode('lineChart', true)"
|
||||
tooltip="Line Chart" tooltip-placement="bottom"><i class="fa fa-line-chart"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default btn-sm"
|
||||
ng-if="paragraph.result.type == 'TABLE'"
|
||||
ng-class="{'active': isGraphMode('scatterChart')}"
|
||||
ng-click="setGraphMode('scatterChart', true)"
|
||||
tooltip="Scatter Chart" tooltip-placement="bottom"><i class="cf cf-scatter-chart"></i>
|
||||
ng-repeat="viz in builtInTableDataVisualizationList track by $index"
|
||||
ng-class="{'active' : isGraphMode(viz.id)}"
|
||||
ng-click="setGraphMode(viz.id, true, false)"
|
||||
tooltip="{{viz.name}}" tooltip-placement="bottom"><i ng-class="viz.icon"></i>
|
||||
</button>
|
||||
|
||||
<button type="button"
|
||||
|
|
|
|||
|
|
@ -14,41 +14,11 @@ limitations under the License.
|
|||
<div id="p{{paragraph.id}}_graph"
|
||||
class="graphContainer"
|
||||
ng-class="{'noOverflow': getGraphMode()=='table'}"
|
||||
ng-if="getResultType()=='TABLE'"
|
||||
ng-show="getResultType()=='TABLE'"
|
||||
>
|
||||
|
||||
<div ng-if="getGraphMode()=='table'"
|
||||
id="p{{paragraph.id}}_table"
|
||||
class="table">
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='multiBarChart'"
|
||||
id="p{{paragraph.id}}_multiBarChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='pieChart'"
|
||||
id="p{{paragraph.id}}_pieChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='stackedAreaChart'"
|
||||
id="p{{paragraph.id}}_stackedAreaChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='lineChart'"
|
||||
id="p{{paragraph.id}}_lineChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='lineWithFocusChart'"
|
||||
id="p{{paragraph.id}}_lineWithFocusChart">
|
||||
<svg></svg>
|
||||
</div>
|
||||
|
||||
<div ng-if="getGraphMode()=='scatterChart'"
|
||||
id="p{{paragraph.id}}_scatterChart">
|
||||
<svg></svg>
|
||||
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
|
||||
id="p{{paragraph.id}}_{{viz.id}}"
|
||||
ng-show="getGraphMode()==viz.id">
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ limitations under the License.
|
|||
<label>
|
||||
<input type="checkbox"
|
||||
ng-model="paragraph.config.graph.lineWithFocus"
|
||||
ng-click="toggleLineWithFocus()" />
|
||||
ng-click="onGraphOptionChange()" />
|
||||
show line chart with focus
|
||||
</label>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -20,11 +20,11 @@ limitations under the License.
|
|||
All fields:
|
||||
<div class="allFields row">
|
||||
<ul class="noDot">
|
||||
<li class="liVertical" ng-repeat="col in paragraph.result.columnNames">
|
||||
<li class="liVertical" ng-repeat="col in tableDataColumns">
|
||||
<div class="btn btn-default btn-xs"
|
||||
data-drag="true"
|
||||
data-jqyoui-options="{revert: 'invalid', helper: 'clone'}"
|
||||
ng-model="paragraph.result.columnNames"
|
||||
ng-model="tableDataColumns"
|
||||
jqyoui-draggable="{index: {{$index}}, placeholder: 'keep'}">
|
||||
{{col.name | limitTo: 30}}{{col.name.length > 30 ? '...' : ''}}
|
||||
</div>
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
165
zeppelin-web/src/app/tabledata/pivot.js
Normal file
165
zeppelin-web/src/app/tabledata/pivot.js
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
var zeppelin = zeppelin || {};
|
||||
|
||||
/**
|
||||
* pivot table data and return d3 chart data
|
||||
*/
|
||||
zeppelin.PivotTransformation = function(config) {
|
||||
zeppelin.Transformation.call(this, config);
|
||||
};
|
||||
|
||||
zeppelin.PivotTransformation.prototype = Object.create(zeppelin.Transformation.prototype);
|
||||
|
||||
/**
|
||||
* Method will be invoked when tableData or config changes
|
||||
*/
|
||||
zeppelin.PivotTransformation.prototype.transform = function(tableData) {
|
||||
return this.pivot(
|
||||
tableData,
|
||||
this.config.keys,
|
||||
this.config.groups,
|
||||
this.config.values);
|
||||
};
|
||||
|
||||
zeppelin.PivotTransformation.prototype.pivot = function(data, keys, groups, values) {
|
||||
var aggrFunc = {
|
||||
sum: function(a, b) {
|
||||
var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
|
||||
var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
|
||||
return varA + varB;
|
||||
},
|
||||
count: function(a, b) {
|
||||
var varA = (a !== undefined) ? parseInt(a) : 0;
|
||||
var varB = (b !== undefined) ? 1 : 0;
|
||||
return varA + varB;
|
||||
},
|
||||
min: function(a, b) {
|
||||
var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
|
||||
var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
|
||||
return Math.min(varA,varB);
|
||||
},
|
||||
max: function(a, b) {
|
||||
var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
|
||||
var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
|
||||
return Math.max(varA,varB);
|
||||
},
|
||||
avg: function(a, b, c) {
|
||||
var varA = (a !== undefined) ? (isNaN(a) ? 1 : parseFloat(a)) : 0;
|
||||
var varB = (b !== undefined) ? (isNaN(b) ? 1 : parseFloat(b)) : 0;
|
||||
return varA + varB;
|
||||
}
|
||||
};
|
||||
|
||||
var aggrFuncDiv = {
|
||||
sum: false,
|
||||
count: false,
|
||||
min: false,
|
||||
max: false,
|
||||
avg: true
|
||||
};
|
||||
|
||||
var schema = {};
|
||||
var rows = {};
|
||||
|
||||
for (var i = 0; i < data.rows.length; i++) {
|
||||
var row = data.rows[i];
|
||||
var s = schema;
|
||||
var p = rows;
|
||||
|
||||
for (var k = 0; k < keys.length; k++) {
|
||||
var key = keys[k];
|
||||
|
||||
// add key to schema
|
||||
if (!s[key.name]) {
|
||||
s[key.name] = {
|
||||
order: k,
|
||||
index: key.index,
|
||||
type: 'key',
|
||||
children: {}
|
||||
};
|
||||
}
|
||||
s = s[key.name].children;
|
||||
|
||||
// add key to row
|
||||
var keyKey = row[key.index];
|
||||
if (!p[keyKey]) {
|
||||
p[keyKey] = {};
|
||||
}
|
||||
p = p[keyKey];
|
||||
}
|
||||
|
||||
for (var g = 0; g < groups.length; g++) {
|
||||
var group = groups[g];
|
||||
var groupKey = row[group.index];
|
||||
|
||||
// add group to schema
|
||||
if (!s[groupKey]) {
|
||||
s[groupKey] = {
|
||||
order: g,
|
||||
index: group.index,
|
||||
type: 'group',
|
||||
children: {}
|
||||
};
|
||||
}
|
||||
s = s[groupKey].children;
|
||||
|
||||
// add key to row
|
||||
if (!p[groupKey]) {
|
||||
p[groupKey] = {};
|
||||
}
|
||||
p = p[groupKey];
|
||||
}
|
||||
|
||||
for (var v = 0; v < values.length; v++) {
|
||||
var value = values[v];
|
||||
var valueKey = value.name + '(' + value.aggr + ')';
|
||||
|
||||
// add value to schema
|
||||
if (!s[valueKey]) {
|
||||
s[valueKey] = {
|
||||
type: 'value',
|
||||
order: v,
|
||||
index: value.index
|
||||
};
|
||||
}
|
||||
|
||||
// add value to row
|
||||
if (!p[valueKey]) {
|
||||
p[valueKey] = {
|
||||
value: (value.aggr !== 'count') ? row[value.index] : 1,
|
||||
count: 1
|
||||
};
|
||||
} else {
|
||||
p[valueKey] = {
|
||||
value: aggrFunc[value.aggr](p[valueKey].value, row[value.index], p[valueKey].count + 1),
|
||||
count: (aggrFuncDiv[value.aggr]) ? p[valueKey].count + 1 : p[valueKey].count
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//console.log('schema=%o, rows=%o', schema, rows);
|
||||
return {
|
||||
keys: keys,
|
||||
groups: groups,
|
||||
values: values,
|
||||
schema: schema,
|
||||
rows: rows
|
||||
};
|
||||
};
|
||||
|
||||
90
zeppelin-web/src/app/tabledata/tabledata.js
Normal file
90
zeppelin-web/src/app/tabledata/tabledata.js
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
var zeppelin = zeppelin || {};
|
||||
|
||||
/**
|
||||
* Create table data object from paragraph table type result
|
||||
*/
|
||||
zeppelin.TableData = function(columns, rows, comment) {
|
||||
this.columns = columns || [];
|
||||
this.rows = rows || [];
|
||||
this.comment = comment || '';
|
||||
};
|
||||
|
||||
zeppelin.TableData.prototype.loadParagraphResult = function(paragraphResult) {
|
||||
if (!paragraphResult || paragraphResult.type !== 'TABLE') {
|
||||
console.log('Can not load paragraph result');
|
||||
return;
|
||||
}
|
||||
|
||||
var columnNames = [];
|
||||
var rows = [];
|
||||
var array = [];
|
||||
var textRows = paragraphResult.msg.split('\n');
|
||||
var comment = '';
|
||||
var commentRow = false;
|
||||
|
||||
for (var i = 0; i < textRows.length; i++) {
|
||||
var textRow = textRows[i];
|
||||
if (commentRow) {
|
||||
comment += textRow;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (textRow === '') {
|
||||
if (rows.length > 0) {
|
||||
commentRow = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
var textCols = textRow.split('\t');
|
||||
var cols = [];
|
||||
var cols2 = [];
|
||||
for (var j = 0; j < textCols.length; j++) {
|
||||
var col = textCols[j];
|
||||
if (i === 0) {
|
||||
columnNames.push({name: col, index: j, aggr: 'sum'});
|
||||
} else {
|
||||
var parsedCol = this.parseTableCell(col);
|
||||
cols.push(parsedCol);
|
||||
cols2.push({key: (columnNames[i]) ? columnNames[i].name : undefined, value: parsedCol});
|
||||
}
|
||||
}
|
||||
if (i !== 0) {
|
||||
rows.push(cols);
|
||||
array.push(cols2);
|
||||
}
|
||||
}
|
||||
this.comment = comment;
|
||||
this.columns = columnNames;
|
||||
this.rows = rows;
|
||||
};
|
||||
|
||||
zeppelin.TableData.prototype.parseTableCell = function(cell) {
|
||||
if (!isNaN(cell)) {
|
||||
if (cell.length === 0 || Number(cell) > Number.MAX_SAFE_INTEGER || Number(cell) < Number.MIN_SAFE_INTEGER) {
|
||||
return cell;
|
||||
} else {
|
||||
return Number(cell);
|
||||
}
|
||||
}
|
||||
var d = moment(cell);
|
||||
if (d.isValid()) {
|
||||
return d;
|
||||
}
|
||||
return cell;
|
||||
};
|
||||
35
zeppelin-web/src/app/tabledata/transformation.js
Normal file
35
zeppelin-web/src/app/tabledata/transformation.js
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
var zeppelin = zeppelin || {};
|
||||
|
||||
/**
|
||||
* Base class for visualization
|
||||
*/
|
||||
zeppelin.Transformation = function(config) {
|
||||
this.config = config;
|
||||
};
|
||||
|
||||
/**
|
||||
* Method will be invoked when tableData or config changes
|
||||
*/
|
||||
zeppelin.Transformation.prototype.transform = function(tableData) {
|
||||
// override this
|
||||
};
|
||||
|
||||
zeppelin.Transformation.prototype.setConfig = function(config) {
|
||||
this.config = config;
|
||||
};
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in area chart
|
||||
*/
|
||||
zeppelin.AreachartVisualization = function(targetEl, config) {
|
||||
zeppelin.Nvd3ChartVisualization.call(this, targetEl, config);
|
||||
|
||||
var PivotTransformation = zeppelin.PivotTransformation;
|
||||
this.pivot = new PivotTransformation(config);
|
||||
this.xLables = [];
|
||||
};
|
||||
|
||||
zeppelin.AreachartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
|
||||
|
||||
zeppelin.AreachartVisualization.prototype.type = function() {
|
||||
return 'stackedAreaChart';
|
||||
};
|
||||
|
||||
zeppelin.AreachartVisualization.prototype.getTransformation = function() {
|
||||
return this.pivot;
|
||||
};
|
||||
|
||||
zeppelin.AreachartVisualization.prototype.render = function(tableData) {
|
||||
var pivot = this.pivot.transform(tableData);
|
||||
var d3Data = this.d3DataFromPivot(
|
||||
pivot.schema,
|
||||
pivot.rows,
|
||||
pivot.keys,
|
||||
pivot.groups,
|
||||
pivot.values,
|
||||
false,
|
||||
true,
|
||||
false);
|
||||
|
||||
this.xLabels = d3Data.xLabels;
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render.call(this, d3Data);
|
||||
};
|
||||
|
||||
/**
|
||||
* Set new config
|
||||
*/
|
||||
zeppelin.AreachartVisualization.prototype.setConfig = function(config) {
|
||||
zeppelin.Nvd3ChartVisualization.prototype.setConfig.call(this, config);
|
||||
this.pivot.setConfig(config);
|
||||
};
|
||||
|
||||
zeppelin.AreachartVisualization.prototype.configureChart = function(chart) {
|
||||
var self = this;
|
||||
chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
|
||||
chart.yAxisTickFormat(function(d) {return self.yAxisTickFormat(d);});
|
||||
chart.yAxis.axisLabelDistance(50);
|
||||
chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)
|
||||
};
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in bar char
|
||||
*/
|
||||
zeppelin.BarchartVisualization = function(targetEl, config) {
|
||||
zeppelin.Nvd3ChartVisualization.call(this, targetEl, config);
|
||||
|
||||
var PivotTransformation = zeppelin.PivotTransformation;
|
||||
this.pivot = new PivotTransformation(config);
|
||||
};
|
||||
|
||||
zeppelin.BarchartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
|
||||
|
||||
zeppelin.BarchartVisualization.prototype.type = function() {
|
||||
return 'multiBarChart';
|
||||
};
|
||||
|
||||
zeppelin.BarchartVisualization.prototype.getTransformation = function() {
|
||||
return this.pivot;
|
||||
};
|
||||
|
||||
zeppelin.BarchartVisualization.prototype.render = function(tableData) {
|
||||
var pivot = this.pivot.transform(tableData);
|
||||
var d3Data = this.d3DataFromPivot(
|
||||
pivot.schema,
|
||||
pivot.rows,
|
||||
pivot.keys,
|
||||
pivot.groups,
|
||||
pivot.values,
|
||||
true,
|
||||
false,
|
||||
true);
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render.call(this, d3Data);
|
||||
};
|
||||
|
||||
/**
|
||||
* Set new config
|
||||
*/
|
||||
zeppelin.BarchartVisualization.prototype.setConfig = function(config) {
|
||||
zeppelin.Nvd3ChartVisualization.prototype.setConfig.call(this, config);
|
||||
this.pivot.setConfig(config);
|
||||
};
|
||||
|
||||
zeppelin.BarchartVisualization.prototype.configureChart = function(chart) {
|
||||
var self = this;
|
||||
chart.yAxis.axisLabelDistance(50);
|
||||
chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d);});
|
||||
};
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in line chart
|
||||
*/
|
||||
zeppelin.LinechartVisualization = function(targetEl, config) {
|
||||
zeppelin.Nvd3ChartVisualization.call(this, targetEl, config);
|
||||
|
||||
var PivotTransformation = zeppelin.PivotTransformation;
|
||||
this.pivot = new PivotTransformation(config);
|
||||
this.xLables = [];
|
||||
};
|
||||
|
||||
zeppelin.LinechartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
|
||||
|
||||
zeppelin.LinechartVisualization.prototype.type = function() {
|
||||
if (this.config.lineWithFocus) {
|
||||
return 'lineWithFocusChart';
|
||||
} else {
|
||||
return 'lineChart';
|
||||
}
|
||||
};
|
||||
|
||||
zeppelin.LinechartVisualization.prototype.getTransformation = function() {
|
||||
return this.pivot;
|
||||
};
|
||||
|
||||
zeppelin.LinechartVisualization.prototype.render = function(tableData) {
|
||||
this.tableData = tableData;
|
||||
var pivot = this.pivot.transform(tableData);
|
||||
var d3Data = this.d3DataFromPivot(
|
||||
pivot.schema,
|
||||
pivot.rows,
|
||||
pivot.keys,
|
||||
pivot.groups,
|
||||
pivot.values,
|
||||
false,
|
||||
true,
|
||||
false);
|
||||
|
||||
this.xLabels = d3Data.xLabels;
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render.call(this, d3Data);
|
||||
};
|
||||
|
||||
/**
|
||||
* Set new config
|
||||
*/
|
||||
zeppelin.LinechartVisualization.prototype.setConfig = function(config) {
|
||||
zeppelin.Nvd3ChartVisualization.prototype.setConfig.call(this, config);
|
||||
this.pivot.setConfig(config);
|
||||
|
||||
// change mode
|
||||
if (this.currentMode !== config.lineWithFocus) {
|
||||
zeppelin.Nvd3ChartVisualization.prototype.destroy.call(this);
|
||||
this.currentMode = config.lineWithFocus;
|
||||
}
|
||||
};
|
||||
|
||||
zeppelin.LinechartVisualization.prototype.configureChart = function(chart) {
|
||||
var self = this;
|
||||
chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
|
||||
chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d, self.xLabels);});
|
||||
chart.yAxis.axisLabelDistance(50);
|
||||
if (chart.useInteractiveGuideline) { // lineWithFocusChart hasn't got useInteractiveGuideline
|
||||
chart.useInteractiveGuideline(true); // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691)
|
||||
}
|
||||
if (this.config.forceY) {
|
||||
chart.forceY([0]); // force y-axis minimum to 0 for line chart.
|
||||
} else {
|
||||
chart.forceY([]);
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,257 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in table format
|
||||
*/
|
||||
zeppelin.Nvd3ChartVisualization = function(targetEl, config) {
|
||||
zeppelin.Visualization.call(this, targetEl, config);
|
||||
this.targetEl.append('<svg></svg>');
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype = Object.create(zeppelin.Visualization.prototype);
|
||||
|
||||
zeppelin.Visualization.prototype.refresh = function() {
|
||||
if (this.chart) {
|
||||
this.chart.update();
|
||||
}
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render = function(data) {
|
||||
var type = this.type();
|
||||
var d3g = data.d3g;
|
||||
|
||||
if (!this.chart) {
|
||||
this.chart = nv.models[type]();
|
||||
}
|
||||
|
||||
this.configureChart(this.chart);
|
||||
|
||||
var animationDuration = 300;
|
||||
var numberOfDataThreshold = 150;
|
||||
var height = this.targetEl.height();
|
||||
|
||||
// turn off animation when dataset is too large. (for performance issue)
|
||||
// still, since dataset is large, the chart content sequentially appears like animated.
|
||||
try {
|
||||
if (d3g[0].values.length > numberOfDataThreshold) {
|
||||
animationDuration = 0;
|
||||
}
|
||||
} catch (ignoreErr) {
|
||||
}
|
||||
|
||||
d3.select('#' + this.targetEl[0].id + ' svg')
|
||||
.attr('height', height)
|
||||
.datum(d3g)
|
||||
.transition()
|
||||
.duration(animationDuration)
|
||||
.call(this.chart);
|
||||
d3.select('#' + this.targetEl[0].id + ' svg').style.height = height + 'px';
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.type = function() {
|
||||
// override this and return chart type
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.configureChart = function(chart) {
|
||||
// override this to configure chart
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.groupedThousandsWith3DigitsFormatter = function(x) {
|
||||
return d3.format(',')(d3.round(x, 3));
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.customAbbrevFormatter = function(x) {
|
||||
var s = d3.format('.3s')(x);
|
||||
switch (s[s.length - 1]) {
|
||||
case 'G': return s.slice(0, -1) + 'B';
|
||||
}
|
||||
return s;
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.xAxisTickFormat = function(d, xLabels) {
|
||||
if (xLabels[d] && (isNaN(parseFloat(xLabels[d])) || !isFinite(xLabels[d]))) { // to handle string type xlabel
|
||||
return xLabels[d];
|
||||
} else {
|
||||
return d;
|
||||
}
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.yAxisTickFormat = function(d) {
|
||||
if (Math.abs(d) >= Math.pow(10,6)) {
|
||||
return this.customAbbrevFormatter(d);
|
||||
}
|
||||
return this.groupedThousandsWith3DigitsFormatter(d);
|
||||
};
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.d3DataFromPivot = function(
|
||||
schema, rows, keys, groups, values, allowTextXAxis, fillMissingValues, multiBarChart) {
|
||||
// construct table data
|
||||
var d3g = [];
|
||||
|
||||
var concat = function(o, n) {
|
||||
if (!o) {
|
||||
return n;
|
||||
} else {
|
||||
return o + '.' + n;
|
||||
}
|
||||
};
|
||||
|
||||
var getSchemaUnderKey = function(key, s) {
|
||||
for (var c in key.children) {
|
||||
s[c] = {};
|
||||
getSchemaUnderKey(key.children[c], s[c]);
|
||||
}
|
||||
};
|
||||
|
||||
var traverse = function(sKey, s, rKey, r, func, rowName, rowValue, colName) {
|
||||
//console.log("TRAVERSE sKey=%o, s=%o, rKey=%o, r=%o, rowName=%o, rowValue=%o, colName=%o", sKey, s, rKey, r, rowName, rowValue, colName);
|
||||
|
||||
if (s.type === 'key') {
|
||||
rowName = concat(rowName, sKey);
|
||||
rowValue = concat(rowValue, rKey);
|
||||
} else if (s.type === 'group') {
|
||||
colName = concat(colName, rKey);
|
||||
} else if (s.type === 'value' && sKey === rKey || valueOnly) {
|
||||
colName = concat(colName, rKey);
|
||||
func(rowName, rowValue, colName, r);
|
||||
}
|
||||
|
||||
for (var c in s.children) {
|
||||
if (fillMissingValues && s.children[c].type === 'group' && r[c] === undefined) {
|
||||
var cs = {};
|
||||
getSchemaUnderKey(s.children[c], cs);
|
||||
traverse(c, s.children[c], c, cs, func, rowName, rowValue, colName);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (var j in r) {
|
||||
if (s.children[c].type === 'key' || c === j) {
|
||||
traverse(c, s.children[c], j, r[j], func, rowName, rowValue, colName);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var valueOnly = (keys.length === 0 && groups.length === 0 && values.length > 0);
|
||||
var noKey = (keys.length === 0);
|
||||
var isMultiBarChart = multiBarChart;
|
||||
|
||||
var sKey = Object.keys(schema)[0];
|
||||
|
||||
var rowNameIndex = {};
|
||||
var rowIdx = 0;
|
||||
var colNameIndex = {};
|
||||
var colIdx = 0;
|
||||
var rowIndexValue = {};
|
||||
|
||||
for (var k in rows) {
|
||||
traverse(sKey, schema[sKey], k, rows[k], function(rowName, rowValue, colName, value) {
|
||||
//console.log("RowName=%o, row=%o, col=%o, value=%o", rowName, rowValue, colName, value);
|
||||
if (rowNameIndex[rowValue] === undefined) {
|
||||
rowIndexValue[rowIdx] = rowValue;
|
||||
rowNameIndex[rowValue] = rowIdx++;
|
||||
}
|
||||
|
||||
if (colNameIndex[colName] === undefined) {
|
||||
colNameIndex[colName] = colIdx++;
|
||||
}
|
||||
var i = colNameIndex[colName];
|
||||
if (noKey && isMultiBarChart) {
|
||||
i = 0;
|
||||
}
|
||||
|
||||
if (!d3g[i]) {
|
||||
d3g[i] = {
|
||||
values: [],
|
||||
key: (noKey && isMultiBarChart) ? 'values' : colName
|
||||
};
|
||||
}
|
||||
|
||||
var xVar = isNaN(rowValue) ? ((allowTextXAxis) ? rowValue : rowNameIndex[rowValue]) : parseFloat(rowValue);
|
||||
var yVar = 0;
|
||||
if (xVar === undefined) { xVar = colName; }
|
||||
if (value !== undefined) {
|
||||
yVar = isNaN(value.value) ? 0 : parseFloat(value.value) / parseFloat(value.count);
|
||||
}
|
||||
d3g[i].values.push({
|
||||
x: xVar,
|
||||
y: yVar
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// clear aggregation name, if possible
|
||||
var namesWithoutAggr = {};
|
||||
var colName;
|
||||
var withoutAggr;
|
||||
// TODO - This part could use som refactoring - Weird if/else with similar actions and variable names
|
||||
for (colName in colNameIndex) {
|
||||
withoutAggr = colName.substring(0, colName.lastIndexOf('('));
|
||||
if (!namesWithoutAggr[withoutAggr]) {
|
||||
namesWithoutAggr[withoutAggr] = 1;
|
||||
} else {
|
||||
namesWithoutAggr[withoutAggr]++;
|
||||
}
|
||||
}
|
||||
|
||||
if (valueOnly) {
|
||||
for (var valueIndex = 0; valueIndex < d3g[0].values.length; valueIndex++) {
|
||||
colName = d3g[0].values[valueIndex].x;
|
||||
if (!colName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
withoutAggr = colName.substring(0, colName.lastIndexOf('('));
|
||||
if (namesWithoutAggr[withoutAggr] <= 1) {
|
||||
d3g[0].values[valueIndex].x = withoutAggr;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (var d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
|
||||
colName = d3g[d3gIndex].key;
|
||||
withoutAggr = colName.substring(0, colName.lastIndexOf('('));
|
||||
if (namesWithoutAggr[withoutAggr] <= 1) {
|
||||
d3g[d3gIndex].key = withoutAggr;
|
||||
}
|
||||
}
|
||||
|
||||
// use group name instead of group.value as a column name, if there're only one group and one value selected.
|
||||
if (groups.length === 1 && values.length === 1) {
|
||||
for (d3gIndex = 0; d3gIndex < d3g.length; d3gIndex++) {
|
||||
colName = d3g[d3gIndex].key;
|
||||
colName = colName.split('.').slice(0, -1).join('.');
|
||||
d3g[d3gIndex].key = colName;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
xLabels: rowIndexValue,
|
||||
d3g: d3g
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* method will be invoked when visualization need to be destroyed.
|
||||
* Don't need to destroy this.targetEl.
|
||||
*/
|
||||
zeppelin.Visualization.prototype.destroy = function() {
|
||||
if (this.chart) {
|
||||
d3.selectAll('#' + this.targetEl[0].id + ' svg > *').remove();
|
||||
this.chart = undefined;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in pie chart
|
||||
*/
|
||||
zeppelin.PiechartVisualization = function(targetEl, config) {
|
||||
zeppelin.Nvd3ChartVisualization.call(this, targetEl, config);
|
||||
|
||||
var PivotTransformation = zeppelin.PivotTransformation;
|
||||
this.pivot = new PivotTransformation(config);
|
||||
};
|
||||
|
||||
zeppelin.PiechartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
|
||||
|
||||
zeppelin.PiechartVisualization.prototype.type = function() {
|
||||
return 'pieChart';
|
||||
};
|
||||
|
||||
zeppelin.PiechartVisualization.prototype.getTransformation = function() {
|
||||
return this.pivot;
|
||||
};
|
||||
|
||||
zeppelin.PiechartVisualization.prototype.render = function(tableData) {
|
||||
var pivot = this.pivot.transform(tableData);
|
||||
var d3Data = this.d3DataFromPivot(
|
||||
pivot.schema,
|
||||
pivot.rows,
|
||||
pivot.keys,
|
||||
pivot.groups,
|
||||
pivot.values,
|
||||
true,
|
||||
false,
|
||||
false);
|
||||
|
||||
var d = d3Data.d3g;
|
||||
var d3g = [];
|
||||
if (d.length > 0) {
|
||||
for (var i = 0; i < d[0].values.length ; i++) {
|
||||
var e = d[0].values[i];
|
||||
d3g.push({
|
||||
label: e.x,
|
||||
value: e.y
|
||||
});
|
||||
}
|
||||
}
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render.call(this, {d3g: d3g});
|
||||
};
|
||||
|
||||
/**
|
||||
* Set new config
|
||||
*/
|
||||
zeppelin.PiechartVisualization.prototype.setConfig = function(config) {
|
||||
zeppelin.Nvd3ChartVisualization.prototype.setConfig.call(this, config);
|
||||
this.pivot.setConfig(config);
|
||||
};
|
||||
|
||||
zeppelin.PiechartVisualization.prototype.configureChart = function(chart) {
|
||||
chart.x(function(d) { return d.label;}).y(function(d) { return d.value;});
|
||||
};
|
||||
|
|
@ -0,0 +1,295 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in scatter char
|
||||
*/
|
||||
zeppelin.ScatterchartVisualization = function(targetEl, config) {
|
||||
zeppelin.Nvd3ChartVisualization.call(this, targetEl, config);
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype = Object.create(zeppelin.Nvd3ChartVisualization.prototype);
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.type = function() {
|
||||
return 'scatterChart';
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.getTransformation = function() {
|
||||
return this.pivot;
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.render = function(tableData) {
|
||||
this.tableData = tableData;
|
||||
var d3Data = this.setScatterChart(tableData, true);
|
||||
this.xLabels = d3Data.xLabels;
|
||||
this.yLabels = d3Data.yLabels;
|
||||
|
||||
zeppelin.Nvd3ChartVisualization.prototype.render.call(this, d3Data);
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.configureChart = function(chart) {
|
||||
var self = this;
|
||||
|
||||
chart.xAxis.tickFormat(function(d) {return self.xAxisTickFormat(d, self.xLabels);});
|
||||
chart.yAxis.tickFormat(function(d) {return self.yAxisTickFormat(d, self.yLabels);});
|
||||
|
||||
// configure how the tooltip looks.
|
||||
chart.tooltipContent(function(key, x, y, graph, data) {
|
||||
var tooltipContent = '<h3>' + key + '</h3>';
|
||||
if (self.config.scatter.size &&
|
||||
self.isValidSizeOption(self.config.scatter, self.tableData.rows)) {
|
||||
tooltipContent += '<p>' + data.point.size + '</p>';
|
||||
}
|
||||
|
||||
return tooltipContent;
|
||||
});
|
||||
|
||||
chart.showDistX(true).showDistY(true);
|
||||
//handle the problem of tooltip not showing when muliple points have same value.
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.setScatterChart = function(data, refresh) {
|
||||
var xAxis = this.config.scatter.xAxis;
|
||||
var yAxis = this.config.scatter.yAxis;
|
||||
var group = this.config.scatter.group;
|
||||
var size = this.config.scatter.size;
|
||||
|
||||
var xValues = [];
|
||||
var yValues = [];
|
||||
var rows = {};
|
||||
var d3g = [];
|
||||
|
||||
var rowNameIndex = {};
|
||||
var colNameIndex = {};
|
||||
var grpNameIndex = {};
|
||||
var rowIndexValue = {};
|
||||
var colIndexValue = {};
|
||||
var grpIndexValue = {};
|
||||
var rowIdx = 0;
|
||||
var colIdx = 0;
|
||||
var grpIdx = 0;
|
||||
var grpName = '';
|
||||
|
||||
var xValue;
|
||||
var yValue;
|
||||
var row;
|
||||
|
||||
if (!xAxis && !yAxis) {
|
||||
return {
|
||||
d3g: []
|
||||
};
|
||||
}
|
||||
|
||||
for (var i = 0; i < data.rows.length; i++) {
|
||||
row = data.rows[i];
|
||||
if (xAxis) {
|
||||
xValue = row[xAxis.index];
|
||||
xValues[i] = xValue;
|
||||
}
|
||||
if (yAxis) {
|
||||
yValue = row[yAxis.index];
|
||||
yValues[i] = yValue;
|
||||
}
|
||||
}
|
||||
|
||||
var isAllDiscrete = ((xAxis && yAxis && this.isDiscrete(xValues) && this.isDiscrete(yValues)) ||
|
||||
(!xAxis && this.isDiscrete(yValues)) ||
|
||||
(!yAxis && this.isDiscrete(xValues)));
|
||||
|
||||
if (isAllDiscrete) {
|
||||
rows = this.setDiscreteScatterData(data);
|
||||
} else {
|
||||
rows = data.rows;
|
||||
}
|
||||
|
||||
if (!group && isAllDiscrete) {
|
||||
grpName = 'count';
|
||||
} else if (!group && !size) {
|
||||
if (xAxis && yAxis) {
|
||||
grpName = '(' + xAxis.name + ', ' + yAxis.name + ')';
|
||||
} else if (xAxis && !yAxis) {
|
||||
grpName = xAxis.name;
|
||||
} else if (!xAxis && yAxis) {
|
||||
grpName = yAxis.name;
|
||||
}
|
||||
} else if (!group && size) {
|
||||
grpName = size.name;
|
||||
}
|
||||
|
||||
for (i = 0; i < rows.length; i++) {
|
||||
row = rows[i];
|
||||
if (xAxis) {
|
||||
xValue = row[xAxis.index];
|
||||
}
|
||||
if (yAxis) {
|
||||
yValue = row[yAxis.index];
|
||||
}
|
||||
if (group) {
|
||||
grpName = row[group.index];
|
||||
}
|
||||
var sz = (isAllDiscrete) ? row[row.length - 1] : ((size) ? row[size.index] : 1);
|
||||
|
||||
if (grpNameIndex[grpName] === undefined) {
|
||||
grpIndexValue[grpIdx] = grpName;
|
||||
grpNameIndex[grpName] = grpIdx++;
|
||||
}
|
||||
|
||||
if (xAxis && rowNameIndex[xValue] === undefined) {
|
||||
rowIndexValue[rowIdx] = xValue;
|
||||
rowNameIndex[xValue] = rowIdx++;
|
||||
}
|
||||
|
||||
if (yAxis && colNameIndex[yValue] === undefined) {
|
||||
colIndexValue[colIdx] = yValue;
|
||||
colNameIndex[yValue] = colIdx++;
|
||||
}
|
||||
|
||||
if (!d3g[grpNameIndex[grpName]]) {
|
||||
d3g[grpNameIndex[grpName]] = {
|
||||
key: grpName,
|
||||
values: []
|
||||
};
|
||||
}
|
||||
|
||||
d3g[grpNameIndex[grpName]].values.push({
|
||||
x: xAxis ? (isNaN(xValue) ? rowNameIndex[xValue] : parseFloat(xValue)) : 0,
|
||||
y: yAxis ? (isNaN(yValue) ? colNameIndex[yValue] : parseFloat(yValue)) : 0,
|
||||
size: isNaN(parseFloat(sz)) ? 1 : parseFloat(sz)
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
xLabels: rowIndexValue,
|
||||
yLabels: colIndexValue,
|
||||
d3g: d3g
|
||||
};
|
||||
};
|
||||
zeppelin.ScatterchartVisualization.prototype.setDiscreteScatterData = function(data) {
|
||||
var xAxis = this.config.scatter.xAxis;
|
||||
var yAxis = this.config.scatter.yAxis;
|
||||
var group = this.config.scatter.group;
|
||||
|
||||
var xValue;
|
||||
var yValue;
|
||||
var grp;
|
||||
|
||||
var rows = {};
|
||||
|
||||
for (var i = 0; i < data.rows.length; i++) {
|
||||
var row = data.rows[i];
|
||||
if (xAxis) {
|
||||
xValue = row[xAxis.index];
|
||||
}
|
||||
if (yAxis) {
|
||||
yValue = row[yAxis.index];
|
||||
}
|
||||
if (group) {
|
||||
grp = row[group.index];
|
||||
}
|
||||
|
||||
var key = xValue + ',' + yValue + ',' + grp;
|
||||
|
||||
if (!rows[key]) {
|
||||
rows[key] = {
|
||||
x: xValue,
|
||||
y: yValue,
|
||||
group: grp,
|
||||
size: 1
|
||||
};
|
||||
} else {
|
||||
rows[key].size++;
|
||||
}
|
||||
}
|
||||
|
||||
// change object into array
|
||||
var newRows = [];
|
||||
for (var r in rows) {
|
||||
var newRow = [];
|
||||
if (xAxis) { newRow[xAxis.index] = rows[r].x; }
|
||||
if (yAxis) { newRow[yAxis.index] = rows[r].y; }
|
||||
if (group) { newRow[group.index] = rows[r].group; }
|
||||
newRow[data.rows[0].length] = rows[r].size;
|
||||
newRows.push(newRow);
|
||||
}
|
||||
return newRows;
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.isDiscrete = function(field) {
|
||||
var getUnique = function(f) {
|
||||
var uniqObj = {};
|
||||
var uniqArr = [];
|
||||
var j = 0;
|
||||
for (var i = 0; i < f.length; i++) {
|
||||
var item = f[i];
|
||||
if (uniqObj[item] !== 1) {
|
||||
uniqObj[item] = 1;
|
||||
uniqArr[j++] = item;
|
||||
}
|
||||
}
|
||||
return uniqArr;
|
||||
};
|
||||
|
||||
for (var i = 0; i < field.length; i++) {
|
||||
if (isNaN(parseFloat(field[i])) &&
|
||||
(typeof field[i] === 'string' || field[i] instanceof String)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
var threshold = 0.05;
|
||||
var unique = getUnique(field);
|
||||
if (unique.length / field.length < threshold) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
zeppelin.ScatterchartVisualization.prototype.isValidSizeOption = function(options) {
|
||||
var xValues = [];
|
||||
var yValues = [];
|
||||
var rows = this.tableData.rows;
|
||||
|
||||
for (var i = 0; i < rows.length; i++) {
|
||||
var row = rows[i];
|
||||
var size = row[options.size.index];
|
||||
|
||||
//check if the field is numeric
|
||||
if (isNaN(parseFloat(size)) || !isFinite(size)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (options.xAxis) {
|
||||
var x = row[options.xAxis.index];
|
||||
xValues[i] = x;
|
||||
}
|
||||
if (options.yAxis) {
|
||||
var y = row[options.yAxis.index];
|
||||
yValues[i] = y;
|
||||
}
|
||||
}
|
||||
|
||||
//check if all existing fields are discrete
|
||||
var isAllDiscrete = ((options.xAxis && options.yAxis && this.isDiscrete(xValues) && this.isDiscrete(yValues)) ||
|
||||
(!options.xAxis && this.isDiscrete(yValues)) ||
|
||||
(!options.yAxis && this.isDiscrete(xValues)));
|
||||
|
||||
if (isAllDiscrete) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Visualize data in table format
|
||||
*/
|
||||
zeppelin.TableVisualization = function(targetEl) {
|
||||
zeppelin.Visualization.call(this, targetEl);
|
||||
console.log('Init table viz');
|
||||
targetEl.addClass('table');
|
||||
};
|
||||
|
||||
zeppelin.TableVisualization.prototype = Object.create(zeppelin.Visualization.prototype);
|
||||
|
||||
zeppelin.TableVisualization.prototype.refresh = function() {
|
||||
this.hot.render();
|
||||
};
|
||||
|
||||
zeppelin.TableVisualization.prototype.render = function(tableData) {
|
||||
var height = this.targetEl.height();
|
||||
var container = this.targetEl.css('height', height).get(0);
|
||||
var resultRows = tableData.rows;
|
||||
var columnNames = _.pluck(tableData.columns, 'name');
|
||||
|
||||
if (this.hot) {
|
||||
this.hot.destroy();
|
||||
}
|
||||
|
||||
this.hot = new Handsontable(container, {
|
||||
colHeaders: columnNames,
|
||||
data: resultRows,
|
||||
rowHeaders: false,
|
||||
stretchH: 'all',
|
||||
sortIndicator: true,
|
||||
columnSorting: true,
|
||||
contextMenu: false,
|
||||
manualColumnResize: true,
|
||||
manualRowResize: true,
|
||||
readOnly: true,
|
||||
readOnlyCellClassName: '', // don't apply any special class so we can retain current styling
|
||||
fillHandle: false,
|
||||
fragmentSelection: true,
|
||||
disableVisualSelection: true,
|
||||
cells: function(row, col, prop) {
|
||||
var cellProperties = {};
|
||||
cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) {
|
||||
if (value instanceof moment) {
|
||||
td.innerHTML = value._i;
|
||||
} else if (!isNaN(value)) {
|
||||
cellProperties.format = '0,0.[00000]';
|
||||
td.style.textAlign = 'left';
|
||||
Handsontable.renderers.NumericRenderer.apply(this, arguments);
|
||||
} else if (value.length > '%html'.length && '%html ' === value.substring(0, '%html '.length)) {
|
||||
td.innerHTML = value.substring('%html'.length);
|
||||
} else {
|
||||
Handsontable.renderers.TextRenderer.apply(this, arguments);
|
||||
}
|
||||
};
|
||||
return cellProperties;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
zeppelin.TableVisualization.prototype.destroy = function() {
|
||||
if (this.hot) {
|
||||
this.hot.destroy();
|
||||
}
|
||||
};
|
||||
103
zeppelin-web/src/app/visualization/visualization.js
Normal file
103
zeppelin-web/src/app/visualization/visualization.js
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
var zeppelin = zeppelin || {};
|
||||
|
||||
/**
|
||||
* Base class for visualization
|
||||
*/
|
||||
zeppelin.Visualization = function(targetEl, config) {
|
||||
this.targetEl = targetEl;
|
||||
this.config = config;
|
||||
this._resized = false;
|
||||
this._active = false;
|
||||
};
|
||||
|
||||
/**
|
||||
* get transformation
|
||||
*/
|
||||
zeppelin.Visualization.prototype.getTransformation = function() {
|
||||
// override this
|
||||
};
|
||||
|
||||
/**
|
||||
* Method will be invoked when data or configuration changed
|
||||
*/
|
||||
zeppelin.Visualization.prototype.render = function(tableData) {
|
||||
// override this
|
||||
};
|
||||
|
||||
/**
|
||||
* Refresh visualization.
|
||||
*/
|
||||
zeppelin.Visualization.prototype.refresh = function() {
|
||||
// override this
|
||||
};
|
||||
|
||||
/**
|
||||
* Activate. invoked when visualization is selected
|
||||
*/
|
||||
zeppelin.Visualization.prototype.activate = function() {
|
||||
console.log('active');
|
||||
if (!this._active && this._resized) {
|
||||
var self = this;
|
||||
// give some time for element ready
|
||||
setTimeout(function() {self.refresh();}, 300);
|
||||
this._resized = false;
|
||||
}
|
||||
this._active = true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Activate. invoked when visualization is de selected
|
||||
*/
|
||||
zeppelin.Visualization.prototype.deactivate = function() {
|
||||
console.log('deactive');
|
||||
this._active = false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Is active
|
||||
*/
|
||||
zeppelin.Visualization.prototype.isActive = function() {
|
||||
return this._active;
|
||||
};
|
||||
|
||||
/**
|
||||
* When window or paragraph is resized
|
||||
*/
|
||||
zeppelin.Visualization.prototype.resize = function() {
|
||||
if (this.isActive()) {
|
||||
this.refresh();
|
||||
} else {
|
||||
this._resized = true;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Set new config
|
||||
*/
|
||||
zeppelin.Visualization.prototype.setConfig = function(config) {
|
||||
this.config = config;
|
||||
};
|
||||
|
||||
/**
|
||||
* method will be invoked when visualization need to be destroyed.
|
||||
* Don't need to destroy this.targetEl.
|
||||
*/
|
||||
zeppelin.Visualization.prototype.destroy = function() {
|
||||
// override this
|
||||
};
|
||||
|
|
@ -100,6 +100,20 @@ limitations under the License.
|
|||
<![endif]-->
|
||||
<!-- endbuild -->
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
extensions: ["tex2jax.js"],
|
||||
jax: ["input/TeX", "output/HTML-CSS"],
|
||||
tex2jax: {
|
||||
inlineMath: [ ["\\(","\\)"] ],
|
||||
displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
|
||||
processEscapes: true
|
||||
},
|
||||
"HTML-CSS": { availableFonts: ["TeX"] },
|
||||
messageStyle: "none"
|
||||
});
|
||||
</script>
|
||||
|
||||
<!-- build:js(.) scripts/vendor.js -->
|
||||
<!-- bower:js -->
|
||||
<script src="bower_components/jquery/dist/jquery.js"></script>
|
||||
|
|
@ -147,12 +161,24 @@ limitations under the License.
|
|||
<script src="bower_components/handsontable/dist/handsontable.js"></script>
|
||||
<script src="bower_components/moment-duration-format/lib/moment-duration-format.js"></script>
|
||||
<script src="bower_components/select2/dist/js/select2.js"></script>
|
||||
<script src="bower_components/MathJax/MathJax.js"></script>
|
||||
<!-- endbower -->
|
||||
<!-- endbuild -->
|
||||
<!-- build:js({.tmp,src}) scripts/scripts.js -->
|
||||
<script src="app/app.js"></script>
|
||||
<script src="app/app.controller.js"></script>
|
||||
<script src="app/home/home.controller.js"></script>
|
||||
<script src="app/tabledata/tabledata.js"></script>
|
||||
<script src="app/tabledata/transformation.js"></script>
|
||||
<script src="app/tabledata/pivot.js"></script>
|
||||
<script src="app/visualization/visualization.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-table.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-nvd3chart.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-barchart.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-piechart.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-areachart.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-linechart.js"></script>
|
||||
<script src="app/visualization/builtins/visualization-scatterchart.js"></script>
|
||||
<script src="app/notebook/notebook.controller.js"></script>
|
||||
<script src="app/jobmanager/jobmanager.controller.js"></script>
|
||||
<script src="app/jobmanager/jobs/job.controller.js"></script>
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@
|
|||
"inject": false,
|
||||
"it": false,
|
||||
"jasmine": false,
|
||||
"spyOn": false
|
||||
"spyOn": false,
|
||||
"zeppelin" : false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -65,17 +65,21 @@ module.exports = function(config) {
|
|||
'bower_components/handsontable/dist/handsontable.js',
|
||||
'bower_components/moment-duration-format/lib/moment-duration-format.js',
|
||||
'bower_components/select2/dist/js/select2.js',
|
||||
'bower_components/MathJax/MathJax.js',
|
||||
'bower_components/angular-mocks/angular-mocks.js',
|
||||
// endbower
|
||||
'src/app/app.js',
|
||||
'src/app/app.controller.js',
|
||||
'src/app/tabledata/transformation.js',
|
||||
'src/app/**/*.js',
|
||||
'src/components/**/*.js',
|
||||
'test/spec/**/*.js'
|
||||
],
|
||||
|
||||
// list of files / patterns to exclude
|
||||
exclude: [],
|
||||
exclude: [
|
||||
'src/app/visualization/builtins/*.js'
|
||||
],
|
||||
|
||||
// web server port
|
||||
port: 9002,
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ describe('Controller: ParagraphCtrl', function() {
|
|||
'closeTable', 'openTable', 'showTitle', 'hideTitle', 'setTitle', 'showLineNumbers', 'hideLineNumbers',
|
||||
'changeColWidth', 'columnWidthClass', 'toggleGraphOption', 'toggleOutput', 'loadForm',
|
||||
'aceChanged', 'aceLoaded', 'getEditorValue', 'getProgress', 'getExecutionTime', 'isResultOutdated',
|
||||
'getResultType', 'loadTableData', 'setGraphMode', 'isGraphMode', 'onGraphOptionChange',
|
||||
'getResultType', 'setGraphMode', 'isGraphMode', 'onGraphOptionChange',
|
||||
'removeGraphOptionKeys', 'removeGraphOptionValues', 'removeGraphOptionGroups', 'setGraphOptionValueAggr',
|
||||
'removeScatterOptionXaxis', 'removeScatterOptionYaxis', 'removeScatterOptionGroup',
|
||||
'removeScatterOptionSize'];
|
||||
|
|
@ -64,10 +64,8 @@ describe('Controller: ParagraphCtrl', function() {
|
|||
scope.getResultType = jasmine.createSpy('getResultType spy').andCallFake(function() {
|
||||
return 'TABLE';
|
||||
});
|
||||
spyOn(scope, 'loadTableData');
|
||||
spyOn(scope, 'setGraphMode');
|
||||
scope.init(paragraphMock);
|
||||
expect(scope.loadTableData).toHaveBeenCalled();
|
||||
expect(scope.setGraphMode).toHaveBeenCalled();
|
||||
expect(scope.getGraphMode()).toEqual('table');
|
||||
});
|
||||
|
|
|
|||
41
zeppelin-web/test/spec/tabledata/tabledata.js
Normal file
41
zeppelin-web/test/spec/tabledata/tabledata.js
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
describe('TableData build', function() {
|
||||
var td;
|
||||
|
||||
beforeEach(function() {
|
||||
var TableData = zeppelin.TableData;
|
||||
td = new TableData();
|
||||
});
|
||||
|
||||
it('should initialize the default value', function() {
|
||||
expect(td.columns.length).toBe(0);
|
||||
expect(td.rows.length).toBe(0);
|
||||
expect(td.comment).toBe('');
|
||||
});
|
||||
|
||||
it('should able to create Tabledata from paragraph result', function() {
|
||||
td.loadParagraphResult({
|
||||
type: 'TABLE',
|
||||
msg: 'key\tvalue\na\t10\nb\t20\n\nhello'
|
||||
});
|
||||
|
||||
expect(td.columns.length).toBe(2);
|
||||
expect(td.rows.length).toBe(2);
|
||||
expect(td.comment).toBe('hello');
|
||||
});
|
||||
});
|
||||
|
|
@ -238,7 +238,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
interpreterInfo =
|
||||
new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
|
||||
r.getEditor());
|
||||
add(r.getGroup(), interpreterInfo, r.getProperties(), r.getPath());
|
||||
add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath());
|
||||
}
|
||||
|
||||
for (String settingId : interpreterSettingsRef.keySet()) {
|
||||
|
|
@ -350,9 +350,11 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
InterpreterInfo interpreterInfo =
|
||||
new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
|
||||
registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
|
||||
|
||||
// use defaultOption if it is not specified in interpreter-setting.json
|
||||
InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption :
|
||||
registeredInterpreter.getOption();
|
||||
add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
|
||||
absolutePath);
|
||||
option, absolutePath);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -617,12 +619,11 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
|
||||
private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
|
||||
Map<String, InterpreterProperty> interpreterProperties, String path)
|
||||
Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path)
|
||||
throws InterpreterException, IOException, RepositoryException {
|
||||
ArrayList<InterpreterInfo> infos = new ArrayList<>();
|
||||
infos.add(interpreterInfo);
|
||||
return add(group, infos, new ArrayList<Dependency>(), defaultOption,
|
||||
interpreterProperties, path);
|
||||
return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ public class ZeppelinhubClient {
|
|||
Session session = future.get();
|
||||
zeppelinSession = ZeppelinhubSession.createInstance(session, zeppelinhubToken);
|
||||
} catch (IOException | InterruptedException | ExecutionException e) {
|
||||
LOG.info("Couldnt connect to zeppelinhub", e);
|
||||
LOG.info("Couldnt connect to zeppelinhub - {}", e.toString());
|
||||
zeppelinSession = ZeppelinhubSession.EMPTY;
|
||||
}
|
||||
return zeppelinSession;
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ public class ZeppelinWebsocket implements WebSocketListener {
|
|||
|
||||
@Override
|
||||
public void onWebSocketError(Throwable e) {
|
||||
LOG.warn("Zeppelin socket connection error ", e);
|
||||
LOG.warn("Zeppelin socket connection error: {}", e.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -67,7 +67,7 @@ public class ZeppelinWebsocket implements WebSocketListener {
|
|||
zeppelinClient.handleMsgFromZeppelin(data, noteId);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.error("Failed to send message to ZeppelinHub: ", e);
|
||||
LOG.error("Failed to send message to ZeppelinHub: {}", e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ public class ZeppelinhubWebsocket implements WebSocketListener {
|
|||
|
||||
@Override
|
||||
public void onWebSocketError(Throwable cause) {
|
||||
LOG.error("Got error", cause);
|
||||
LOG.error("Remote websocket error");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
Loading…
Reference in a new issue