mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
swakrish
commit
f8bf1f3516
32 changed files with 1173 additions and 444 deletions
|
|
@ -1,6 +1,6 @@
|
|||
#Zeppelin
|
||||
|
||||
**Documentation:** [User Guide](http://zeppelin.incubator.apache.org/docs/index.html)<br/>
|
||||
**Documentation:** [User Guide](http://zeppelin.incubator.apache.org/docs/latest/index.html)<br/>
|
||||
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.incubator.apache.org/community.html)<br/>
|
||||
**Continuous Integration:** [](https://travis-ci.org/apache/incubator-zeppelin) <br/>
|
||||
**Contributing:** [Contribution Guide](https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md)<br/>
|
||||
|
|
@ -45,6 +45,7 @@ sudo ln -s /usr/local/apache-maven-3.3.3/bin/mvn /usr/local/bin/mvn
|
|||
_Notes:_
|
||||
- Ensure node is installed by running `node --version`
|
||||
- Ensure maven is running version 3.1.x or higher with `mvn -version`
|
||||
- Configure maven to use more memory than usual by ```export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=1024m"```
|
||||
|
||||
### Build
|
||||
If you want to build Zeppelin from the source, please first clone this repository, then:
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
|
|||
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
|
||||
fi
|
||||
|
||||
# Add jdbc connector jar
|
||||
# ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar"
|
||||
|
||||
addJarInDir "${ZEPPELIN_HOME}"
|
||||
addJarInDir "${ZEPPELIN_HOME}/lib"
|
||||
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreters</name>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter</value>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter</value>
|
||||
<description>Comma separated interpreter configurations. First interpreter become a default</description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
|
||||
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/cassandra.html">Cassandra</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 322 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 298 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 396 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 146 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 257 KiB |
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_user.png
Normal file
BIN
docs/assets/themes/zeppelin/img/docs-img/zeppelin_user.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 325 KiB |
|
|
@ -32,10 +32,8 @@ All Interpreters in the same interpreter group are launched in a single, separat
|
|||
### Make your own Interpreter
|
||||
|
||||
Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods.
|
||||
|
||||
You can include org.apache.zeppelin:zeppelin-interpreter:[VERSION] artifact in your build system.
|
||||
|
||||
Your interpreter name is derived from the static register method
|
||||
You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system.
|
||||
Your interpreter name is derived from the static register method.
|
||||
|
||||
```
|
||||
static {
|
||||
|
|
@ -44,16 +42,15 @@ static {
|
|||
```
|
||||
|
||||
The name will appear later in the interpreter name option box during the interpreter configuration process.
|
||||
|
||||
The name of the interpreter is what you later write to identify a paragraph which should be interpreted using this interpreter.
|
||||
|
||||
```
|
||||
%MyInterpreterName
|
||||
some interpreter spesific code...
|
||||
some interpreter specific code...
|
||||
```
|
||||
### Install your interpreter binary
|
||||
|
||||
Once you have build your interpreter, you can place your interpreter under directory with all the dependencies.
|
||||
Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies.
|
||||
|
||||
```
|
||||
[ZEPPELIN_HOME]/interpreter/[INTERPRETER_NAME]/
|
||||
|
|
@ -63,33 +60,34 @@ Once you have build your interpreter, you can place your interpreter under direc
|
|||
|
||||
To configure your interpreter you need to follow these steps:
|
||||
|
||||
1. create conf/zeppelin-site.xml by copying conf/zeppelin-site.xml.template to conf/zeppelin-site.xml
|
||||
1. Add your interpreter class name to the zeppelin.interpreters property in `conf/zeppelin-site.xml`.
|
||||
|
||||
2. Add your interpreter class name to the zeppelin.interpreters property in conf/zeppelin-site.xml
|
||||
|
||||
Property value is comma separated [INTERPRETER_CLASS_NAME]
|
||||
for example,
|
||||
Property value is comma separated [INTERPRETER\_CLASS\_NAME].
|
||||
For example,
|
||||
|
||||
```
|
||||
```
|
||||
<property>
|
||||
<name>zeppelin.interpreters</name>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter</value>
|
||||
</property>
|
||||
```
|
||||
3. start zeppelin by running ```./bin/zeppelin-deamon start```
|
||||
|
||||
4. in the interpreter page, click the +Create button and configure your interpreter properties.
|
||||
2. Add your interpreter to the [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `zeppelin-site.xml`.
|
||||
|
||||
3. Start Zeppelin by running `./bin/zeppelin-daemon.sh start`.
|
||||
|
||||
4. In the interpreter page, click the `+Create` button and configure your interpreter properties.
|
||||
Now you are done and ready to use your interpreter.
|
||||
|
||||
Note that the interpreters shipped with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no zeppelin-site.xml.
|
||||
Note that the interpreters shipped with zeppelin have a [default configuration](https://github.com/apache/incubator-zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`.
|
||||
|
||||
### Use your interpreter
|
||||
|
||||
#### 0.5.0
|
||||
Inside of a notebook, %[INTERPRETER_NAME] directive will call your interpreter.
|
||||
Inside of a notebook, `%[INTERPRETER_NAME]` directive will call your interpreter.
|
||||
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
|
||||
|
||||
for example
|
||||
For example,
|
||||
|
||||
```
|
||||
%myintp
|
||||
|
|
@ -100,16 +98,14 @@ println(a)
|
|||
|
||||
<br />
|
||||
#### 0.6.0 and later
|
||||
Inside of a notebook, %[INTERPRETER\_GROUP].[INTERPRETER\_NAME] directive will call your interpreter.
|
||||
Inside of a notebook, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter.
|
||||
Note that the first interpreter configuration in zeppelin.interpreters will be the default one.
|
||||
|
||||
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. Omit [INTERPRETER\_NAME] selects first available interpreter in the [INTERPRETER\_GROUP].
|
||||
Omit '[INTERPRETER\_GROUP]' will selects [INTERPRETER\_NAME] from default interpreter group.
|
||||
You can omit either [INTERPRETER\_GROUP] or [INTERPRETER\_NAME]. If you omit [INTERPRETER\_NAME], then first available interpreter will be selected in the [INTERPRETER\_GROUP].
|
||||
Likewise, if you skip [INTERPRETER\_GROUP], then [INTERPRETER\_NAME] will be chosen from default interpreter group.
|
||||
|
||||
|
||||
For example, if you have two interpreter myintp1 and myintp2 in group mygrp,
|
||||
|
||||
you can call myintp1 like
|
||||
For example, if you have two interpreter myintp1 and myintp2 in group mygrp, you can call myintp1 like
|
||||
|
||||
```
|
||||
%mygrp.myintp1
|
||||
|
|
@ -125,7 +121,7 @@ and you can call myintp2 like
|
|||
codes for myintp2
|
||||
```
|
||||
|
||||
If you omit your interpreter name, it'll selects first available interpreter in the group (myintp1)
|
||||
If you omit your interpreter name, it'll select first available interpreter in the group ( myintp1 ).
|
||||
|
||||
```
|
||||
%mygrp
|
||||
|
|
|
|||
|
|
@ -6,10 +6,8 @@ group: manual
|
|||
---
|
||||
{% include JB/setup %}
|
||||
|
||||
<hr/>
|
||||
## 1. Cassandra CQL Interpreter for Apache Zeppelin
|
||||
|
||||
<br/>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
|
|
@ -23,35 +21,32 @@ group: manual
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 2. Enabling Cassandra Interpreter
|
||||
|
||||
In a notebook, to enable the **Cassandra** interpreter, click on the **Gear** icon and select **Cassandra**
|
||||
In a notebook, to enable the **Cassandra** interpreter, click on the **Gear** icon and select **Cassandra**.
|
||||
|
||||
<center>
|
||||
|
||||
|
||||
|
||||
<center>
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/cassandra-InterpreterBinding.png" width="25%" height="25%">
|
||||
<br>
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/cassandra-InterpreterSelection.png" width="25%" height="25%">
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
|
||||
|
||||
## 3. Using the Cassandra Interpreter
|
||||
|
||||
In a paragraph, use **_%cassandra_** to select the **Cassandra** interpreter and then input all commands.
|
||||
|
||||
To access the interactive help, type **HELP;**
|
||||
To access the interactive help, type `HELP;`
|
||||
|
||||
<center>
|
||||
|
||||
<center>
|
||||
<img src="../assets/themes/zeppelin/img/docs-img/cassandra-InteractiveHelp.png" width="50%" height="50%">
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 4. Interpreter Commands
|
||||
|
||||
The **Cassandra** interpreter accepts the following commands
|
||||
The **Cassandra** interpreter accepts the following commands.
|
||||
|
||||
<center>
|
||||
<table class="table-configuration">
|
||||
|
|
@ -85,10 +80,9 @@ group: manual
|
|||
<td>All CQL-compatible statements (SELECT, INSERT, CREATE ...)</td>
|
||||
<td>All CQL statements are executed directly against the Cassandra server</td>
|
||||
</tr>
|
||||
</table>
|
||||
</table>
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
## 5. CQL statements
|
||||
|
||||
This interpreter is compatible with any CQL statement supported by Cassandra. Ex:
|
||||
|
|
@ -97,7 +91,7 @@ This interpreter is compatible with any CQL statement supported by Cassandra. Ex
|
|||
|
||||
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
|
||||
SELECT * FROM users WHERE login='jdoe';
|
||||
```
|
||||
```
|
||||
|
||||
Each statement should be separated by a semi-colon ( **;** ) except the special commands below:
|
||||
|
||||
|
|
@ -149,7 +143,7 @@ This means that the following statements are equivalent and valid:
|
|||
```
|
||||
|
||||
The complete list of all CQL statements and versions can be found below:
|
||||
<center>
|
||||
<center>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Cassandra Version</th>
|
||||
|
|
@ -163,7 +157,7 @@ The complete list of all CQL statements and versions can be found below:
|
|||
http://docs.datastax.com/en/cql/3.3/cql/cqlIntro.html
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>2.1 & 2.0</strong></td>
|
||||
<td>
|
||||
|
|
@ -172,7 +166,7 @@ The complete list of all CQL statements and versions can be found below:
|
|||
http://docs.datastax.com/en/cql/3.1/cql/cql_intro_c.html
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>1.2</strong></td>
|
||||
<td>
|
||||
|
|
@ -181,11 +175,10 @@ The complete list of all CQL statements and versions can be found below:
|
|||
http://docs.datastax.com/en/cql/3.0/cql/aboutCQL.html
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</tr>
|
||||
</table>
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 6. Comments in statements
|
||||
|
||||
|
|
@ -203,21 +196,19 @@ It is possible to add comments between statements. Single line comments start wi
|
|||
Insert into users(login,name) vAlues('hsue','Helen SUE');
|
||||
```
|
||||
|
||||
<hr/>
|
||||
|
||||
## 7. Syntax Validation
|
||||
|
||||
The interpreters is shipped with a built-in syntax validator. This validator only checks for basic syntax errors.
|
||||
All CQL-related syntax validation is delegated directly to **Cassandra**
|
||||
All CQL-related syntax validation is delegated directly to **Cassandra**.
|
||||
|
||||
Most of the time, syntax errors are due to **missing semi-colons** between statements or **typo errors**.
|
||||
|
||||
<hr/>
|
||||
|
||||
## 8. Schema commands
|
||||
|
||||
To make schema discovery easier and more interactive, the following commands are supported:
|
||||
<center>
|
||||
<center>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Command</th>
|
||||
|
|
@ -226,67 +217,65 @@ To make schema discovery easier and more interactive, the following commands are
|
|||
<tr>
|
||||
<td><strong>DESCRIBE CLUSTER;</strong></td>
|
||||
<td>Show the current cluster name and its partitioner</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>DESCRIBE KEYSPACES;</strong></td>
|
||||
<td>List all existing keyspaces in the cluster and their configuration (replication factor, durable write ...)</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>DESCRIBE TABLES;</strong></td>
|
||||
<td>List all existing keyspaces in the cluster and for each, all the tables name</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>DESCRIBE TYPES;</strong></td>
|
||||
<td>List all existing user defined types in the <strong>current (logged) keyspace</strong></td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE FUNCTIONS <keyspace_name>;</strong></td>
|
||||
<td>List all existing user defined functions in the given keyspace</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE AGGREGATES <keyspace_name>;</strong></td>
|
||||
<td>List all existing user defined aggregates in the given keyspace</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE KEYSPACE <keyspace_name>;</strong></td>
|
||||
<td>Describe the given keyspace configuration and all its table details (name, columns, ...)</td>
|
||||
</tr>
|
||||
<td>Describe the given keyspace configuration and all its table details (name, columns, ...)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE TABLE (<keyspace_name>).<table_name>;</strong></td>
|
||||
<td>
|
||||
Describe the given table. If the keyspace is not provided, the current logged in keyspace is used.
|
||||
If there is no logged in keyspace, the default system keyspace is used.
|
||||
If no table is found, an error message is raised
|
||||
If no table is found, an error message is raised.
|
||||
</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE TYPE (<keyspace_name>).<type_name>;</strong></td>
|
||||
<td>
|
||||
Describe the given type(UDT). If the keyspace is not provided, the current logged in keyspace is used.
|
||||
If there is no logged in keyspace, the default system keyspace is used.
|
||||
If no type is found, an error message is raised
|
||||
If no type is found, an error message is raised.
|
||||
</td>
|
||||
</tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE FUNCTION (<keyspace_name>).<function_name>;</strong></td>
|
||||
<td>Describe the given user defined function. The keyspace is optional</td>
|
||||
</tr>
|
||||
<td>Describe the given user defined function. The keyspace is optional.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><strong>DESCRIBE AGGREGATE (<keyspace_name>).<aggregate_name>;</strong></td>
|
||||
<td>Describe the given user defined aggregate. The keyspace is optional</td>
|
||||
</tr>
|
||||
<td>Describe the given user defined aggregate. The keyspace is optional.</td>
|
||||
</tr>
|
||||
</table>
|
||||
</center>
|
||||
</center>
|
||||
|
||||
The schema objects (cluster, keyspace, table, type, function and aggregate) are displayed in a tabular format.
|
||||
There is a drop-down menu on the top left corner to expand objects details. On the top right menu is shown the Icon legend.
|
||||
|
||||
<br/>
|
||||
<center>
|
||||
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 9. Runtime Parameters
|
||||
|
||||
|
|
@ -294,8 +283,7 @@ Sometimes you want to be able to pass runtime query parameters to your statement
|
|||
Those parameters are not part of the CQL specs and are specific to the interpreter.
|
||||
Below is the list of all parameters:
|
||||
|
||||
<br/>
|
||||
<center>
|
||||
<center>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
|
|
@ -305,38 +293,37 @@ Below is the list of all parameters:
|
|||
<tr>
|
||||
<td nowrap>Consistency Level</td>
|
||||
<td><strong>@consistency=<em>value</em></strong></td>
|
||||
<td>Apply the given consistency level to all queries in the paragraph</td>
|
||||
<td>Apply the given consistency level to all queries in the paragraph.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Serial Consistency Level</td>
|
||||
<td><strong>@serialConsistency=<em>value</em></strong></td>
|
||||
<td>Apply the given serial consistency level to all queries in the paragraph</td>
|
||||
<td>Apply the given serial consistency level to all queries in the paragraph.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Timestamp</td>
|
||||
<td><strong>@timestamp=<em>long value</em></strong></td>
|
||||
<td>
|
||||
Apply the given timestamp to all queries in the paragraph.
|
||||
Please note that timestamp value passed directly in CQL statement will override this value
|
||||
Please note that timestamp value passed directly in CQL statement will override this value.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Retry Policy</td>
|
||||
<td><strong>@retryPolicy=<em>value</em></strong></td>
|
||||
<td>Apply the given retry policy to all queries in the paragraph</td>
|
||||
<td>Apply the given retry policy to all queries in the paragraph.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Fetch Size</td>
|
||||
<td><strong>@fetchSize=<em>integer value</em></strong></td>
|
||||
<td>Apply the given fetch size to all queries in the paragraph</td>
|
||||
<td>Apply the given fetch size to all queries in the paragraph.</td>
|
||||
</tr>
|
||||
</table>
|
||||
</center>
|
||||
|
||||
Some parameters only accept restricted values:
|
||||
|
||||
<br/>
|
||||
<center>
|
||||
<center>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
|
|
@ -344,11 +331,11 @@ Below is the list of all parameters:
|
|||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Consistency Level</td>
|
||||
<td><strong>ALL, ANY, ONE, TWO, THREE, QUORUM, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM</strong></td>
|
||||
<td><strong>ALL, ANY, ONE, TWO, THREE, QUORUM, LOCAL\_ONE, LOCAL\_QUORUM, EACH\_QUORUM</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Serial Consistency Level</td>
|
||||
<td><strong>SERIAL, LOCAL_SERIAL</strong></td>
|
||||
<td><strong>SERIAL, LOCAL\_SERIAL</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Timestamp</td>
|
||||
|
|
@ -356,7 +343,7 @@ Below is the list of all parameters:
|
|||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Retry Policy</td>
|
||||
<td><strong>DEFAULT, DOWNGRADING_CONSISTENCY, FALLTHROUGH, LOGGING_DEFAULT, LOGGING_DOWNGRADING, LOGGING_FALLTHROUGH</strong></td>
|
||||
<td><strong>DEFAULT, DOWNGRADING\_CONSISTENCY, FALLTHROUGH, LOGGING\_DEFAULT, LOGGING\_DOWNGRADING, LOGGING\_FALLTHROUGH</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap>Fetch Size</td>
|
||||
|
|
@ -365,7 +352,7 @@ Below is the list of all parameters:
|
|||
</table>
|
||||
</center>
|
||||
|
||||
>Please note that you should **not** add semi-colon ( **;** ) at the end of each parameter statement
|
||||
>Please note that you should **not** add semi-colon ( **;** ) at the end of each parameter statement.
|
||||
|
||||
Some examples:
|
||||
|
||||
|
|
@ -395,12 +382,11 @@ Some examples:
|
|||
|
||||
Some remarks about query parameters:
|
||||
|
||||
> 1. **many** query parameters can be set in the same paragraph
|
||||
> 2. if the **same** query parameter is set many time with different values, the interpreter only take into account the first value
|
||||
> 3. each query parameter applies to **all CQL statements** in the same paragraph, unless you override the option using plain CQL text (like forcing timestamp with the USING clause)
|
||||
> 4. the order of each query parameter with regard to CQL statement does not matter
|
||||
> 1. **Many** query parameters can be set in the same paragraph.
|
||||
> 2. If the **same** query parameter is set many time with different values, the interpreter only take into account the first value.
|
||||
> 3. Each query parameter applies to **all CQL statements** in the same paragraph, unless you override the option using plain CQL text. ( Like forcing timestamp with the USING clause )
|
||||
> 4. The order of each query parameter with regard to CQL statement does not matter.
|
||||
|
||||
<hr/>
|
||||
|
||||
## 10. Support for Prepared Statements
|
||||
|
||||
|
|
@ -424,17 +410,15 @@ Example:
|
|||
@remove_prepare[statement_name]
|
||||
```
|
||||
|
||||
<br/>
|
||||
#### a. @prepare
|
||||
<br/>
|
||||
You can use the syntax _"@prepare[statement_name]=SELECT ..."_ to create a prepared statement.
|
||||
The _statement_name_ is **mandatory** because the interpreter prepares the given statement with the Java driver and
|
||||
saves the generated prepared statement in an **internal hash map**, using the provided _statement_name_ as search key.
|
||||
You can use the syntax `@prepare[statement_name]=SELECT ...` to create a prepared statement.
|
||||
The `statement_name` is **mandatory** because the interpreter prepares the given statement with the Java driver and
|
||||
saves the generated prepared statement in an **internal hash map**, using the provided `statement_name` as search key.
|
||||
|
||||
> Please note that this internal prepared statement map is shared with **all notebooks** and **all paragraphs** because
|
||||
there is only one instance of the interpreter for Cassandra
|
||||
there is only one instance of the interpreter for Cassandra.
|
||||
|
||||
> If the interpreter encounters **many** @prepare for the **same _statement_name_ (key)**, only the **first** statement will be taken into account.
|
||||
> If the interpreter encounters **many** @prepare for the **same statement_name (key)**, only the **first** statement will be taken into account.
|
||||
|
||||
Example:
|
||||
|
||||
|
|
@ -443,37 +427,36 @@ Example:
|
|||
@prepare[select]=SELECT * FROM spark_demo.albums LIMIT ?
|
||||
|
||||
@prepare[select]=SELECT * FROM spark_demo.artists LIMIT ?
|
||||
```
|
||||
```
|
||||
|
||||
For the above example, the prepared statement is _SELECT * FROM spark_demo.albums LIMIT ?_.
|
||||
_SELECT * FROM spark_demo.artists LIMIT ?_ is ignored because an entry already exists in the prepared statements map with the key select.
|
||||
For the above example, the prepared statement is `SELECT * FROM spark_demo.albums LIMIT ?`.
|
||||
`SELECT * FROM spark_demo.artists LIMIT ?` is ignored because an entry already exists in the prepared statements map with the key select.
|
||||
|
||||
In the context of **Zeppelin**, a notebook can be scheduled to be executed at regular interval,
|
||||
thus it is necessary to **avoid re-preparing many time the same statement (considered an anti-pattern)**.
|
||||
<br/>
|
||||
<br/>
|
||||
|
||||
#### b. @bind
|
||||
<br/>
|
||||
Once the statement is prepared (possibly in a separated notebook/paragraph). You can bind values to it:
|
||||
|
||||
Once the statement is prepared ( possibly in a separated notebook/paragraph ). You can bind values to it:
|
||||
|
||||
```
|
||||
@bind[select_first]=10
|
||||
```
|
||||
```
|
||||
|
||||
Bound values are not mandatory for the **@bind** statement. However if you provide bound values, they need to comply to some syntax:
|
||||
Bound values are not mandatory for the `@bind` statement. However if you provide bound values, they need to comply to some syntax:
|
||||
|
||||
* String values should be enclosed between simple quotes ( ‘ )
|
||||
* Date values should be enclosed between simple quotes ( ‘ ) and respect the formats:
|
||||
1. yyyy-MM-dd HH:MM:ss
|
||||
2. yyyy-MM-dd HH:MM:ss.SSS
|
||||
* **null** is parsed as-is
|
||||
* **boolean** (true|false) are parsed as-is
|
||||
* **null** is parsed as-is.
|
||||
* **boolean** (true|false) is parsed as-is.
|
||||
* collection values must follow the **[standard CQL syntax]**:
|
||||
* list: [‘list_item1’, ’list_item2’, ...]
|
||||
* set: {‘set_item1’, ‘set_item2’, …}
|
||||
* map: {‘key1’: ‘val1’, ‘key2’: ‘val2’, …}
|
||||
* **tuple** values should be enclosed between parenthesis (see **[Tuple CQL syntax]**): (‘text’, 123, true)
|
||||
* **udt** values should be enclosed between brackets (see **[UDT CQL syntax]**): {stree_name: ‘Beverly Hills’, number: 104, zip_code: 90020, state: ‘California’, …}
|
||||
* **tuple** values should be enclosed between parenthesis ( see **[Tuple CQL syntax]** ): (‘text’, 123, true)
|
||||
* **udt** values should be enclosed between brackets ( see **[UDT CQL syntax]** ): {stree_name: ‘Beverly Hills’, number: 104, zip_code: 90020, state: ‘California’, …}
|
||||
|
||||
> It is possible to use the @bind statement inside a batch:
|
||||
>
|
||||
|
|
@ -485,14 +468,12 @@ Bound values are not mandatory for the **@bind** statement. However if you provi
|
|||
> APPLY BATCH;
|
||||
> ```
|
||||
|
||||
<br/>
|
||||
#### c. @remove_prepare
|
||||
<br/>
|
||||
|
||||
To avoid for a prepared statement to stay forever in the prepared statement map, you can use the
|
||||
**@remove_prepare[statement_name]** syntax to remove it.
|
||||
`@remove_prepare[statement_name]` syntax to remove it.
|
||||
Removing a non-existing prepared statement yields no error.
|
||||
|
||||
<hr/>
|
||||
|
||||
## 11. Using Dynamic Forms
|
||||
|
||||
|
|
@ -529,7 +510,6 @@ It is also possible to use dynamic forms for **prepared statements**:
|
|||
|
||||
{% endraw %}
|
||||
|
||||
<hr/>
|
||||
|
||||
## 12. Execution parallelism and shared states
|
||||
|
||||
|
|
@ -543,44 +523,34 @@ Consequently, if you use the **USE _keyspace name_;** statement to log into a ke
|
|||
per instance of **Cassandra** interpreter.
|
||||
|
||||
The same remark does apply to the **prepared statement hash map**, it is shared by **all users** using the same instance of **Cassandra** interpreter.
|
||||
|
||||
Until **Zeppelin** offers a real multi-users separation, there is a work-around to segregate user environment and states:
|
||||
_create different **Cassandra** interpreter instances_
|
||||
create different **Cassandra** interpreter instances.
|
||||
|
||||
For this, first go to the **Interpreter** menu and click on the **Create** button
|
||||
<br/>
|
||||
<br/>
|
||||
For this, first go to the **Interpreter** menu and click on the **Create** button.
|
||||
<center>
|
||||
|
||||
</center>
|
||||
|
||||
In the interpreter creation form, put **cass-instance2** as **Name** and select the **cassandra**
|
||||
in the interpreter drop-down list
|
||||
<br/>
|
||||
<br/>
|
||||
in the interpreter drop-down list
|
||||
<center>
|
||||
|
||||
</center>
|
||||
</center>
|
||||
|
||||
Click on **Save** to create the new interpreter instance. Now you should be able to see it in the interpreter list.
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
<center>
|
||||
|
||||
</center>
|
||||
</center>
|
||||
|
||||
Go back to your notebook and click on the **Gear** icon to configure interpreter bindings.
|
||||
You should be able to see and select the **cass-instance2** interpreter instance in the available
|
||||
interpreter list instead of the standard **cassandra** instance.
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
<center>
|
||||
|
||||
</center>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 13. Interpreter Configuration
|
||||
|
||||
|
|
@ -638,7 +608,7 @@ Below are the configuration parameters and their default value.
|
|||
<strong>
|
||||
It is strongly recommended to let the default value
|
||||
and prefix the table name with the actual keyspace
|
||||
in all of your queries
|
||||
in all of your queries.
|
||||
</strong>
|
||||
</td>
|
||||
<td>system</td>
|
||||
|
|
@ -649,7 +619,7 @@ Below are the configuration parameters and their default value.
|
|||
Load balancing policy. Default = <em>new TokenAwarePolicy(new DCAwareRoundRobinPolicy())</em>
|
||||
To Specify your own policy, provide the <strong>fully qualify class name (FQCN)</strong> of your policy.
|
||||
At runtime the interpreter will instantiate the policy using
|
||||
<strong>Class.forName(FQCN)</strong>
|
||||
<strong>Class.forName(FQCN).</strong>
|
||||
</td>
|
||||
<td>DEFAULT</td>
|
||||
</tr>
|
||||
|
|
@ -723,7 +693,7 @@ Below are the configuration parameters and their default value.
|
|||
<td>
|
||||
Cassandra query default consistency level
|
||||
<br/>
|
||||
Available values: ONE, TWO, THREE, QUORUM, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM, ALL
|
||||
Available values: ONE, TWO, THREE, QUORUM, LOCAL\_ONE, LOCAL\_QUORUM, EACH\_QUORUM, ALL
|
||||
</td>
|
||||
<td>ONE</td>
|
||||
</tr>
|
||||
|
|
@ -748,7 +718,7 @@ Below are the configuration parameters and their default value.
|
|||
Default = new ExponentialReconnectionPolicy(1000, 10 * 60 * 1000)
|
||||
To Specify your own policy, provide the <strong>fully qualify class name (FQCN)</strong> of your policy.
|
||||
At runtime the interpreter will instantiate the policy using
|
||||
<strong>Class.forName(FQCN)</strong>
|
||||
<strong>Class.forName(FQCN).</strong>
|
||||
</td>
|
||||
<td>DEFAULT</td>
|
||||
</tr>
|
||||
|
|
@ -759,7 +729,7 @@ Below are the configuration parameters and their default value.
|
|||
Default = DefaultRetryPolicy.INSTANCE
|
||||
To Specify your own policy, provide the <strong>fully qualify class name (FQCN)</strong> of your policy.
|
||||
At runtime the interpreter will instantiate the policy using
|
||||
<strong>Class.forName(FQCN)</strong>
|
||||
<strong>Class.forName(FQCN).</strong>
|
||||
</td>
|
||||
<td>DEFAULT</td>
|
||||
</tr>
|
||||
|
|
@ -785,18 +755,17 @@ Below are the configuration parameters and their default value.
|
|||
Default = NoSpeculativeExecutionPolicy.INSTANCE
|
||||
To Specify your own policy, provide the <strong>fully qualify class name (FQCN)</strong> of your policy.
|
||||
At runtime the interpreter will instantiate the policy using
|
||||
<strong>Class.forName(FQCN)</strong>
|
||||
<strong>Class.forName(FQCN).</strong>
|
||||
</td>
|
||||
<td>DEFAULT</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<hr/>
|
||||
|
||||
## 14. Bugs & Contacts
|
||||
|
||||
If you encounter a bug for this interpreter, please create a **[JIRA]** ticket and ping me on Twitter
|
||||
at **[@doanduyhai]**
|
||||
at **[@doanduyhai]**.
|
||||
|
||||
|
||||
[Cassandra Java Driver]: https://github.com/datastax/java-driver
|
||||
|
|
|
|||
|
|
@ -8,10 +8,11 @@ group: manual
|
|||
|
||||
|
||||
## Elasticsearch Interpreter for Apache Zeppelin
|
||||
[Elasticsearch](https://www.elastic.co/products/elasticsearch) is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements.
|
||||
|
||||
### 1. Configuration
|
||||
<br />
|
||||
## 1. Configuration
|
||||
|
||||
<br/>
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Property</th>
|
||||
|
|
@ -31,7 +32,7 @@ group: manual
|
|||
<tr>
|
||||
<td>elasticsearch.port</td>
|
||||
<td>9300</td>
|
||||
<td>Connection port <b>(important: this is not the HTTP port, but the transport port)</b></td>
|
||||
<td>Connection port <b>( Important: this is not the HTTP port, but the transport port )</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>elasticsearch.result.size</td>
|
||||
|
|
@ -45,22 +46,17 @@ group: manual
|
|||
</center>
|
||||
|
||||
|
||||
> Note #1: you can add more properties to configure the Elasticsearch client.
|
||||
> **Note #1 :** You can add more properties to configure the Elasticsearch client.
|
||||
|
||||
> Note #2: if you use Shield, you can add a property named `shield.user` with a value containing the name and the password (format: `username:password`). For more details about Shield configuration, consult the [Shield reference guide](https://www.elastic.co/guide/en/shield/current/_using_elasticsearch_java_clients_with_shield.html). Do not forget, to copy the shield client jar in the interpreter directory (`ZEPPELIN_HOME/interpreters/elasticsearch`).
|
||||
> **Note #2 :** If you use Shield, you can add a property named `shield.user` with a value containing the name and the password ( format: `username:password` ). For more details about Shield configuration, consult the [Shield reference guide](https://www.elastic.co/guide/en/shield/current/_using_elasticsearch_java_clients_with_shield.html). Do not forget, to copy the shield client jar in the interpreter directory (`ZEPPELIN_HOME/interpreters/elasticsearch`).
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
### 2. Enabling the Elasticsearch Interpreter
|
||||
<br />
|
||||
## 2. Enabling the Elasticsearch Interpreter
|
||||
|
||||
In a notebook, to enable the **Elasticsearch** interpreter, click the **Gear** icon and select **Elasticsearch**.
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
|
||||
### 3. Using the Elasticsearch Interpreter
|
||||
<br />
|
||||
## 3. Using the Elasticsearch Interpreter
|
||||
|
||||
In a paragraph, use `%elasticsearch` to select the Elasticsearch interpreter and then input all commands. To get the list of available commands, use `help`.
|
||||
|
||||
|
|
@ -82,14 +78,14 @@ Commands:
|
|||
. same comments as for the search
|
||||
- get /index/type/id
|
||||
- delete /index/type/id
|
||||
- index /ndex/type/id <json-formatted document>
|
||||
- index /index/type/id <json-formatted document>
|
||||
. the id can be omitted, elasticsearch will generate one
|
||||
```
|
||||
|
||||
> Tip: use (CTRL + .) for completion
|
||||
> **Tip :** Use ( Ctrl + . ) for autocompletion.
|
||||
|
||||
|
||||
#### get
|
||||
### Get
|
||||
With the `get` command, you can find a document by id. The result is a JSON document.
|
||||
|
||||
```bash
|
||||
|
|
@ -101,12 +97,12 @@ Example:
|
|||
|
||||
|
||||
|
||||
#### search
|
||||
### Search
|
||||
With the `search` command, you can send a search query to Elasticsearch. There are two formats of query:
|
||||
|
||||
* You can provide a JSON-formatted query, that is exactly what you provide when you use the REST API of Elasticsearch.
|
||||
* See [Elasticsearch search API reference document](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) for more details about the content of the search queries.
|
||||
* You can also provide the content of a `query_string`
|
||||
* You can also provide the content of a `query_string`.
|
||||
* This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }`
|
||||
* See [Elasticsearch query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) for more details about the content of such a query.
|
||||
|
||||
|
|
@ -134,7 +130,7 @@ Examples:
|
|||
|
||||
```bash
|
||||
| %elasticsearch
|
||||
| search / { "query": { "match_all": {} } }
|
||||
| search / { "query": { "match_all": { } } }
|
||||
|
|
||||
| %elasticsearch
|
||||
| search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
|
||||
|
|
@ -159,7 +155,7 @@ Examples:
|
|||
| search /logs (404 AND (POST OR DELETE))
|
||||
```
|
||||
|
||||
> **Important**: a document in Elasticsearch is a JSON document, so it is hierarchical, not flat as a row in a SQL table.
|
||||
> **Important** : a document in Elasticsearch is a JSON document, so it is hierarchical, not flat as a row in a SQL table.
|
||||
For the Elastic interpreter, the result of a search query is flattened.
|
||||
|
||||
Suppose we have a JSON document:
|
||||
|
|
@ -179,12 +175,10 @@ Suppose we have a JSON document:
|
|||
|
||||
The data will be flattened like this:
|
||||
|
||||
|
||||
content_length | date | request.headers[0] | request.headers[1] | request.method | request.url | status
|
||||
---------------|------|--------------------|--------------------|----------------|-------------|-------
|
||||
1234 | 2015-12-08T21:03:13.588Z | Accept: \*.\* | Host: apache.org | GET | /zeppelin/4cd001cd-c517-4fa9-b8e5-a06b8f4056c4 | 403
|
||||
|
||||
|
||||
Examples:
|
||||
|
||||
* With a table containing the results:
|
||||
|
|
@ -206,7 +200,7 @@ Examples:
|
|||
|
||||
|
||||
|
||||
#### count
|
||||
### Count
|
||||
With the `count` command, you can count documents available in some indices and types. You can also provide a query.
|
||||
|
||||
```bash
|
||||
|
|
@ -223,7 +217,7 @@ Examples:
|
|||
|
||||
|
||||
|
||||
#### index
|
||||
### Index
|
||||
With the `index` command, you can insert/update a document in Elasticsearch.
|
||||
|
||||
```bash
|
||||
|
|
@ -234,7 +228,7 @@ With the `index` command, you can insert/update a document in Elasticsearch.
|
|||
| index /index/type <JSON document>
|
||||
```
|
||||
|
||||
#### delete
|
||||
### Delete
|
||||
With the `delete` command, you can delete a document.
|
||||
|
||||
```bash
|
||||
|
|
@ -243,14 +237,13 @@ With the `delete` command, you can delete a document.
|
|||
```
|
||||
|
||||
|
||||
### Apply Zeppelin Dynamic Forms
|
||||
|
||||
#### Apply Zeppelin Dynamic Forms
|
||||
|
||||
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features
|
||||
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
|
||||
|
||||
```bash
|
||||
| %elasticsearch
|
||||
| size ${limit=10}
|
||||
| search /index/type { "query": { "match_all": {} } }
|
||||
| search /index/type { "query": { "match_all": { } } }
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ group: manual
|
|||
|
||||
|
||||
## Flink interpreter for Apache Zeppelin
|
||||
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing.
|
||||
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing. Flink’s core is a streaming dataflow engine that provides data distribution, communication, and fault tolerance for distributed computations over data streams. Flink also builds batch processing on top of the streaming engine, overlaying native iteration support, managed memory, and program optimization.
|
||||
|
||||
|
||||
### How to start local Flink cluster, to test the interpreter
|
||||
<br>
|
||||
## How to start local Flink cluster, to test the interpreter
|
||||
Zeppelin comes with pre-configured flink-local interpreter, which starts Flink in a local mode on your machine, so you do not need to install anything.
|
||||
|
||||
### How to configure interpreter to point to Flink cluster
|
||||
## How to configure interpreter to point to Flink cluster
|
||||
At the "Interpreters" menu, you have to create a new Flink interpreter and provide next properties:
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -33,18 +33,13 @@ At the "Interpreters" menu, you have to create a new Flink interpreter and provi
|
|||
<td>6123</td>
|
||||
<td>port of running JobManager</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>xxx</td>
|
||||
<td>yyy</td>
|
||||
<td>anything else from [Flink Configuration](https://ci.apache.org/projects/flink/flink-docs-release-0.9/setup/config.html)</td>
|
||||
</tr>
|
||||
</table>
|
||||
<br />
|
||||
|
||||
For more information about Flink configuration, you can find it [here](https://ci.apache.org/projects/flink/flink-docs-release-0.10/setup/config.html).
|
||||
|
||||
### How to test it's working
|
||||
## How to test it's working
|
||||
|
||||
In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL05GTGFicy96ZXBwZWxpbi1ub3RlYm9va3MvbWFzdGVyL25vdGVib29rcy8yQVFFREs1UEMvbm90ZS5qc29u) is from [Till Rohrmann's presentation](http://www.slideshare.net/tillrohrmann/data-analysis-49806564) "Interactive data analysis with Apache Flink" for Apache Flink Meetup.
|
||||
In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL05GTGFicy96ZXBwZWxpbi1ub3RlYm9va3MvbWFzdGVyL25vdGVib29rcy8yQVFFREs1UEMvbm90ZS5qc29u) is from Till Rohrmann's presentation [Interactive data analysis with Apache Flink](http://www.slideshare.net/tillrohrmann/data-analysis-49806564) for Apache Flink Meetup.
|
||||
|
||||
|
||||
```
|
||||
|
|
@ -52,7 +47,7 @@ In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/
|
|||
rm 10.txt.utf-8
|
||||
wget http://www.gutenberg.org/ebooks/10.txt.utf-8
|
||||
```
|
||||
```
|
||||
{% highlight scala %}
|
||||
%flink
|
||||
case class WordCount(word: String, frequency: Int)
|
||||
val bible:DataSet[String] = env.readTextFile("10.txt.utf-8")
|
||||
|
|
@ -65,4 +60,4 @@ val wordCounts = partialCounts.groupBy("word").reduce{
|
|||
(left, right) => WordCount(left.word, left.frequency + right.frequency)
|
||||
}
|
||||
val result10 = wordCounts.first(10).collect()
|
||||
```
|
||||
{% endhighlight %}
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ group: manual
|
|||
|
||||
|
||||
## Hive Interpreter for Apache Zeppelin
|
||||
The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL.
|
||||
|
||||
### Configuration
|
||||
|
||||
<br/>
|
||||
## 1. Configuration
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Property</th>
|
||||
|
|
@ -31,48 +31,48 @@ group: manual
|
|||
<tr>
|
||||
<td>default.user</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Username of the connection</td>
|
||||
<td><b>( Optional ) </b>Username of the connection</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>default.password</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Password of the connection</td>
|
||||
<td><b>( Optional ) </b>Password of the connection</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>default.xxx</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Other properties used by the driver</td>
|
||||
<td><b>( Optional ) </b>Other properties used by the driver</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.driver</td>
|
||||
<td></td>
|
||||
<td>Driver class path of `%hive(${prefix})`</td>
|
||||
<td>Driver class path of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.url</td>
|
||||
<td></td>
|
||||
<td>Url of `%hive(${prefix})`</td>
|
||||
<td>Url of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.user</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Username of the connection of `%hive(${prefix})`</td>
|
||||
<td><b>( Optional ) </b>Username of the connection of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.password</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Password of the connection of `%hive(${prefix})`</td>
|
||||
<td><b>( Optional ) </b>Password of the connection of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>${prefix}.xxx</td>
|
||||
<td></td>
|
||||
<td><b>(Optional)</b>Other properties used by the driver of `%hive(${prefix})`</td>
|
||||
<td><b>( Optional ) </b>Other properties used by the driver of <code>%hive(${prefix})</code> </td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
This interpreter provides multiple configuration with ${prefix}. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
|
||||
This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
|
||||
|
||||
### How to use
|
||||
## 2. How to use
|
||||
|
||||
Basically, you can use
|
||||
|
||||
|
|
@ -91,9 +91,9 @@ select * from my_table;
|
|||
|
||||
You can also run multiple queries up to 10 by default. Changing these settings is not implemented yet.
|
||||
|
||||
#### Apply Zeppelin Dynamic Forms
|
||||
### Apply Zeppelin Dynamic Forms
|
||||
|
||||
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features
|
||||
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
|
||||
|
||||
```sql
|
||||
%hive
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ In order to use Lens interpreters, you may install Apache Lens in some simple st
|
|||
```
|
||||
|
||||
### Configuring Lens Interpreter
|
||||
At the "Interpreters" menu, you can to edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
|
||||
At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
|
||||
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -10,13 +10,14 @@ group: manual
|
|||
|
||||
### Overview
|
||||
[Markdown](http://daringfireball.net/projects/markdown/) is a plain text formatting syntax designed so that it can be converted to HTML.
|
||||
Zeppelin uses markdown4j, for more examples and extension support checkout [markdown4j](https://code.google.com/p/markdown4j/)
|
||||
In Zeppelin notebook you can use ``` %md ``` in the beginning of a paragraph to invoke the Markdown interpreter to generate static html from Markdown plain text.
|
||||
Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/).
|
||||
In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to invoke the Markdown interpreter and generate static html from Markdown plain text.
|
||||
|
||||
In Zeppelin, Markdown interpreter is enabled by default.
|
||||
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="600px" />
|
||||
|
||||
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-interpreter-setting.png" width="60%" />
|
||||
|
||||
### Example
|
||||
The following example demonstrates the basic usage of Markdown in a Zeppelin notebook.
|
||||
|
||||
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="800px" />
|
||||
<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/markdown-example.png" width="70%" />
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ group: manual
|
|||
{% include JB/setup %}
|
||||
|
||||
|
||||
## Spark Interpreter
|
||||
## Spark Interpreter for Apache Zeppelin
|
||||
|
||||
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
|
||||
Spark Interpreter group, which consisted of 4 interpreters.
|
||||
|
|
@ -40,18 +40,15 @@ Spark Interpreter group, which consisted of 4 interpreters.
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
<br />
|
||||
## Configuration
|
||||
|
||||
<br /><br />
|
||||
|
||||
### Configuration
|
||||
<hr />
|
||||
|
||||
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need following two simple steps.
|
||||
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
|
||||
|
||||
|
||||
#### 1. export SPARK_HOME
|
||||
### 1. Export SPARK_HOME
|
||||
|
||||
In **conf/zeppelin-env.sh**, export SPARK_HOME environment variable with your Spark installation path.
|
||||
In **conf/zeppelin-env.sh**, export `SPARK_HOME` environment variable with your Spark installation path.
|
||||
|
||||
for example
|
||||
|
||||
|
|
@ -66,9 +63,7 @@ export HADOOP_CONF_DIR=/usr/lib/hadoop
|
|||
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"
|
||||
```
|
||||
|
||||
|
||||
<br />
|
||||
#### 2. set master in Interpreter menu.
|
||||
### 2. Set master in Interpreter menu
|
||||
|
||||
After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type.
|
||||
|
||||
|
|
@ -81,30 +76,24 @@ for example,
|
|||
|
||||
|
||||
|
||||
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. ( Zeppelin 0.5.5-incubating release works up to Spark 1.5.2 )
|
||||
|
||||
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
|
||||
|
||||
<br />
|
||||
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuild Zeppelin in this way. (Zeppelin 0.5.5-incubating release works up to Spark 1.5.1)
|
||||
|
||||
Note that without exporting SPARK_HOME, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
|
||||
|
||||
<br /> <br />
|
||||
### SparkContext, SQLContext, ZeppelinContext
|
||||
<hr />
|
||||
|
||||
## SparkContext, SQLContext, ZeppelinContext
|
||||
SparkContext, SQLContext, ZeppelinContext are automatically created and exposed as variable names 'sc', 'sqlContext' and 'z', respectively, both in scala and python environments.
|
||||
|
||||
Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
|
||||
|
||||
> Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
|
||||
|
||||
<br />
|
||||
<a name="dependencyloading"> </a>
|
||||
<br />
|
||||
<br />
|
||||
### Dependency Management
|
||||
<hr />
|
||||
There are two ways to load external library in spark interpreter. First is using Zeppelin's %dep interpreter and second is loading Spark properties.
|
||||
## Dependency Management
|
||||
There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties.
|
||||
|
||||
#### 1. Dynamic Dependency Loading via %dep interpreter
|
||||
### 1. Dynamic Dependency Loading via %dep interpreter
|
||||
|
||||
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using %dep interpreter.
|
||||
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
|
||||
|
||||
* Load libraries recursively from Maven repository
|
||||
* Load libraries from local filesystem
|
||||
|
|
@ -112,7 +101,7 @@ When your code requires external library, instead of doing download/copy/restart
|
|||
* Automatically add libraries to SparkCluster (You can turn off)
|
||||
|
||||
Dep interpreter leverages scala environment. So you can write any Scala code here.
|
||||
Note that %dep interpreter should be used before %spark, %pyspark, %sql.
|
||||
Note that `%dep` interpreter should be used before `%spark`, `%pyspark`, `%sql`.
|
||||
|
||||
Here's usages.
|
||||
|
||||
|
|
@ -150,9 +139,7 @@ z.load("groupId:artifactId:version").exclude("groupId:*")
|
|||
z.load("groupId:artifactId:version").local()
|
||||
```
|
||||
|
||||
|
||||
<br />
|
||||
#### 2. Loading Spark Properties
|
||||
### 2. Loading Spark Properties
|
||||
Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are:
|
||||
|
||||
<table class="table-configuration">
|
||||
|
|
@ -181,9 +168,8 @@ Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit`
|
|||
<td>Comma-separated list of files to be placed in the working directory of each executor.</td>
|
||||
</tr>
|
||||
</table>
|
||||
Note that adding jar to pyspark is only availabe via %dep interpreter at the moment
|
||||
> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment.
|
||||
|
||||
<br/>
|
||||
Here are few examples:
|
||||
|
||||
* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh
|
||||
|
|
@ -197,40 +183,43 @@ Here are few examples:
|
|||
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
|
||||
|
||||
<br />
|
||||
<br />
|
||||
### ZeppelinContext
|
||||
<hr />
|
||||
## ZeppelinContext
|
||||
|
||||
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
|
||||
|
||||
<br />
|
||||
#### Object exchange
|
||||
### Object Exchange
|
||||
|
||||
ZeppelinContext extends map and it's shared between scala, python environment.
|
||||
So you can put some object from scala and read it from python, vise versa.
|
||||
<div class="codetabs">
|
||||
<div data-lang="scala" markdown="1">
|
||||
|
||||
Put object from scala
|
||||
|
||||
```scala
|
||||
{% highlight scala %}
|
||||
// Put object from scala
|
||||
%spark
|
||||
val myObject = ...
|
||||
z.put("objName", myObject)
|
||||
```
|
||||
{% endhighlight %}
|
||||
|
||||
Get object from python
|
||||
</div>
|
||||
<div data-lang="python" markdown="1">
|
||||
|
||||
```python
|
||||
%python
|
||||
{% highlight python %}
|
||||
# Get object from python
|
||||
%pyspark
|
||||
myObject = z.get("objName")
|
||||
```
|
||||
|
||||
<br />
|
||||
#### Form creation
|
||||
{% endhighlight %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
### Form Creation
|
||||
|
||||
ZeppelinContext provides functions for creating forms.
|
||||
In scala and python environments, you can create forms programmatically.
|
||||
<div class="codetabs">
|
||||
<div data-lang="scala" markdown="1">
|
||||
|
||||
```scala
|
||||
{% highlight scala %}
|
||||
%spark
|
||||
/* Create text input form */
|
||||
z.input("formName")
|
||||
|
|
@ -245,7 +234,30 @@ z.select("formName", Seq(("option1", "option1DisplayName"),
|
|||
/* Create select form with default value*/
|
||||
z.select("formName", "option1", Seq(("option1", "option1DisplayName"),
|
||||
("option2", "option2DisplayName")))
|
||||
```
|
||||
{% endhighlight %}
|
||||
|
||||
</div>
|
||||
<div data-lang="python" markdown="1">
|
||||
|
||||
{% highlight python %}
|
||||
%pyspark
|
||||
# Create text input form
|
||||
z.input("formName")
|
||||
|
||||
# Create text input form with default value
|
||||
z.input("formName", "defaultValue")
|
||||
|
||||
# Create select form
|
||||
z.select("formName", [("option1", "option1DisplayName"),
|
||||
("option2", "option2DisplayName")])
|
||||
|
||||
# Create select form with default value
|
||||
z.select("formName", [("option1", "option1DisplayName"),
|
||||
("option2", "option2DisplayName")], "option1")
|
||||
{% endhighlight %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
In sql environment, you can create form in simple template.
|
||||
|
||||
|
|
|
|||
124
docs/manual/dynamicinterpreterload.md
Normal file
124
docs/manual/dynamicinterpreterload.md
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Dynamic Interpreter Loading"
|
||||
description: ""
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
## Dynamic Interpreter Loading using REST API
|
||||
|
||||
Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal).
|
||||
Before we start, if you are not familiar with the concept of **Zeppelin interpreter**, you can check out [Overview of Zeppelin interpreter](../manual/interpreters.html) first.
|
||||
|
||||
<br/>
|
||||
## Overview
|
||||
In the past, Zeppelin was loading interpreter binaries from `/interpreter/[interpreter_name]` directory. They were configured by `zeppelin.interpreters` property in `conf/zeppelin-site.xml` or `ZEPPELIN_INTERPRETERS` env variables in `conf/zeppelin-env.sh`. They were loaded on Zeppelin server startup and stayed alive until the server was stopped.
|
||||
In order to simplify using 3rd party interpreters, we changed this way to **dynamically** load interpreters from **Maven Repository** using **REST API**. Hopefully, the picture below will help you to understand the process.
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin_user.png" height="85%" width="85%"></center>
|
||||
|
||||
## Load & Unload Interpreters Using REST API
|
||||
|
||||
### 1. Load
|
||||
You can **load** interpreters located in Maven repository using REST API, like this:
|
||||
|
||||
( Maybe, you are unfamiliar with `[interpreter_group_name]` or `[interpreter_name]`. If so, please checkout [Interpreters in Zeppelin](../manual/interpreter.html) again. )
|
||||
|
||||
```
|
||||
http://[zeppelin-server]:[zeppelin-port]/api/interpreter/load/[interpreter_group_name]/[interpreter_name]
|
||||
```
|
||||
The Restful method will be <code>**POST**</code>. And the parameters you need are:
|
||||
|
||||
1. **Artifact:** Maven artifact ( groupId:artifactId:version )
|
||||
|
||||
2. **Class Name:** Package name + Interpreter class name
|
||||
|
||||
3. **Repository ( optional ):** Additional maven repository address
|
||||
|
||||
For example, if you want to load `markdown` interpreter to your Zeppelin, the parameters and URL you need may look like:
|
||||
|
||||
```
|
||||
http://127.0.0.1:8080/api/interpreter/load/md/markdown
|
||||
```
|
||||
|
||||
```
|
||||
{
|
||||
"artifact": "org.apache.zeppelin:zeppelin-markdown:0.6.0-incubating-SNAPSHOT",
|
||||
"className": "org.apache.zeppelin.markdown.Markdown",
|
||||
"repository": {
|
||||
"url": "http://dl.bintray.com/spark-packages/maven",
|
||||
"snapshot": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
The meaning of each parameters is:
|
||||
|
||||
1. **Artifact**
|
||||
- groupId: org.apache.zeppelin
|
||||
- artifactId: zeppelin-markdown
|
||||
- version: 0.6.0-incubating-SNAPSHOT
|
||||
|
||||
2. **Class Name**
|
||||
- Package Name: org.apache.zeppelin
|
||||
- Interpreter Class Name: markdown.Markdown
|
||||
|
||||
3. **Repository ( optional )**
|
||||
- Url: http://dl.bintray.com/spark-packages/maven
|
||||
- Snapshot: false
|
||||
|
||||
> <b>Please note: </b>The interpreters you downloaded need to be **reload**, when your Zeppelin server is down.
|
||||
|
||||
### 2. Unload
|
||||
If you want to **unload** the interpreters using REST API,
|
||||
|
||||
```
|
||||
http://[zeppelin-server]:[zeppelin-port]/api/interpreter/unload/[interpreter_group_name]/[interpreter_name]
|
||||
```
|
||||
In this case, the Restful method will be <code>**DELETE**</code>.
|
||||
|
||||
<br/>
|
||||
## What is the next step after Loading ?
|
||||
|
||||
### Q1. Where is the location of interpreters you downloaded ?
|
||||
|
||||
Actually, the answer about this question is in the above picture. Once the REST API is called, the `.jar` files of interpreters you get are saved under `ZEPPELIN_HOME/local-repo` first. Then, they will be copied to `ZEPPELIN_HOME/interpreter` directory. So, please checkout your `ZEPPELIN_HOME/interpreter`.
|
||||
|
||||
### Q2. Then, how can I use this interpreter ?
|
||||
|
||||
After loading an interpreter, you can use it by creating and configuring it in Zeppelin's **Interpreter tab**.
|
||||
|
||||
Oh, you don't need to restart your Zeppelin server. Because it is **Dynamic Loading**, you can configure and load it **at runtime** !
|
||||
|
||||
1. After Zeppelin server up, browse Zeppelin home and click **Interpreter tab**.
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_1.png" height="85%" width="85%"></center>
|
||||
|
||||
2. At the **Interpreter** section, click **+Create** button.
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_2.png" height="85%" width="85%"></center>
|
||||
|
||||
3. Then, you can verify the interpreter list that you loaded.
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_setting_3.png" height="85%" width="85%"></center>
|
||||
|
||||
4. After choosing an interpreter, you can configure and use it. Don't forget to save it.
|
||||
|
||||
5. Create a new notebook in the **Notebook** section, then you can bind the interpreters from your interpreter list. Just drag and drop !
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_binding_1.png" height="85%" width="85%"></center>
|
||||
<center><img src="../assets/themes/zeppelin/img/docs-img/interpreter_binding_2.png" height="85%" width="85%"></center>
|
||||
|
||||
6. At last, you can use your interpreter !
|
||||
|
||||
If you want to get the specific information about respective interpreters, please checkout each interpreter documentation.
|
||||
|
|
@ -20,45 +20,50 @@ limitations under the License.
|
|||
{% include JB/setup %}
|
||||
|
||||
|
||||
## Interpreters in zeppelin
|
||||
## Interpreters in Zeppelin
|
||||
In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin.
|
||||
The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin.
|
||||
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, Hive, Markdown, Shell and so on.
|
||||
|
||||
This section explain the role of Interpreters, interpreters group and interpreters settings in Zeppelin.
|
||||
Zeppelin interpreter concept allows any language/data-processing-backend to be plugged into Zeppelin.
|
||||
Currently Zeppelin supports many interpreters such as Scala(with Apache Spark), Python(with Apache Spark), SparkSQL, Hive, Markdown and Shell.
|
||||
<br/>
|
||||
## What is Zeppelin interpreter?
|
||||
|
||||
### What is zeppelin interpreter?
|
||||
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use scala code in Zeppelin, you need `%spark` interpreter.
|
||||
|
||||
Zeppelin Interpreter is the plug-in which enable zeppelin user to use a specific language/data-processing-backend. For example to use scala code in Zeppelin, you need ```spark``` interpreter.
|
||||
|
||||
When you click on the ```+Create``` button in the interpreter page the interpreter drop-down list box will present all the available interpreters on your server.
|
||||
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
|
||||
|
||||
### What is zeppelin interpreter setting?
|
||||
<br/>
|
||||
## What is Zeppelin Interpreter Setting?
|
||||
|
||||
Zeppelin interpreter setting is the configuration of a given interpreter on zeppelin server. For example, the properties requried for hive JDBC interpreter to connect to the Hive server.
|
||||
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
|
||||
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
|
||||
### What is zeppelin interpreter group?
|
||||
|
||||
Every Interpreter belongs to an InterpreterGroup. InterpreterGroup is a unit of start/stop interpreter.
|
||||
By default, every interpreter belong to a single group but the group might contain more interpreters. For example, spark interpreter group include spark support, pySpark,
|
||||
<br/>
|
||||
## What is Zeppelin Interpreter Group?
|
||||
|
||||
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
|
||||
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, spark interpreter group is including Spark support, pySpark,
|
||||
SparkSQL and the dependency loader.
|
||||
|
||||
Technically, Zeppelin interpreters from the same group are running in the same JVM.
|
||||
Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html).
|
||||
|
||||
Interpreters belong to a single group a registered together and all of their properties are listed in the interpreter setting.
|
||||
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
|
||||
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
|
||||
|
||||
### Programming langages for interpreter
|
||||
<br/>
|
||||
## Programming Languages for Interpreter
|
||||
|
||||
If the interpreter uses a specific programming language (like Scala, Python, SQL), it is generally a good idea to add syntax highlighting support for that to the notebook paragraph editor.
|
||||
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
|
||||
|
||||
To check out the list of languages supported, see the mode-*.js files under zeppelin-web/bower_components/ace-builds/src-noconflict or from github https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict
|
||||
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
|
||||
|
||||
To add a new set of syntax highlighting,
|
||||
1. add the mode-*.js file to zeppelin-web/bower.json (when built, zeppelin-web/src/index.html will be changed automatically)
|
||||
2. add to the list of `editorMode` in zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js - it follows the pattern 'ace/mode/x' where x is the name
|
||||
3. add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` in zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
|
||||
If you want to add a new set of syntax highlighting,
|
||||
|
||||
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
|
||||
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
|
||||
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,28 +22,29 @@ limitations under the License.
|
|||
## Zeppelin REST API
|
||||
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
|
||||
|
||||
All REST API are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
|
||||
|
||||
All REST API are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
|
||||
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON view such as
|
||||
[JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
|
||||
[JSON View](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
|
||||
|
||||
|
||||
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
|
||||
If you work with zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.incubator.apache.org/community.html).
|
||||
|
||||
<br />
|
||||
### Interpreter REST API list
|
||||
## Interpreter REST API List
|
||||
|
||||
The role of registered interpreters, settings and interpreters group is described [here](../manual/interpreters.html)
|
||||
The role of registered interpreters, settings and interpreters group are described in [here](../manual/interpreters.html).
|
||||
|
||||
### 1. List of Registered Interpreters & Interpreter Settings
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<th>List registered interpreters</th>
|
||||
<th>List of registered interpreters</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method return all the registered interpreters available on the server.</td>
|
||||
<td>This ```GET``` method returns all the registered interpreters available on the server.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
|
|
@ -54,12 +55,11 @@ limitations under the License.
|
|||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -113,12 +113,12 @@ limitations under the License.
|
|||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
<th>List interpreters settings</th>
|
||||
<th>List of interpreters settings</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```GET``` method return all the interpreters settings registered on the server.</td>
|
||||
<td>This ```GET``` method returns all the interpreters settings registered on the server.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
|
|
@ -129,12 +129,11 @@ limitations under the License.
|
|||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -182,7 +181,8 @@ limitations under the License.
|
|||
</table>
|
||||
|
||||
<br/>
|
||||
|
||||
### 2. Create an Interpreter Setting
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -202,12 +202,11 @@ limitations under the License.
|
|||
<td>201</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON input
|
||||
</td>
|
||||
<td>Sample JSON input</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -227,8 +226,7 @@ limitations under the License.
|
|||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -256,7 +254,8 @@ limitations under the License.
|
|||
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
### 3. Update an Interpreter Setting
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -276,12 +275,11 @@ limitations under the License.
|
|||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON input
|
||||
</td>
|
||||
<td>Sample JSON input</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -301,8 +299,7 @@ limitations under the License.
|
|||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>
|
||||
{
|
||||
|
|
@ -330,7 +327,8 @@ limitations under the License.
|
|||
|
||||
|
||||
<br/>
|
||||
|
||||
### 4. Delete an Interpreter Setting
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -354,17 +352,17 @@ limitations under the License.
|
|||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>{"status":"OK"}</pre>
|
||||
<code>{"status":"OK"}</code>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
<br/>
|
||||
|
||||
### 5. Restart an Interpreter
|
||||
|
||||
<table class="table-configuration">
|
||||
<col width="200">
|
||||
<tr>
|
||||
|
|
@ -373,7 +371,7 @@ limitations under the License.
|
|||
</tr>
|
||||
<tr>
|
||||
<td>Description</td>
|
||||
<td>This ```PUT``` method restart the given interpreter id.</td>
|
||||
<td>This ```PUT``` method restarts the given interpreter id.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>URL</td>
|
||||
|
|
@ -384,14 +382,13 @@ limitations under the License.
|
|||
<td>200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> Fail code</td>
|
||||
<td>Fail code</td>
|
||||
<td> 500 </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> sample JSON response
|
||||
</td>
|
||||
<td>Sample JSON response</td>
|
||||
<td>
|
||||
<pre>{"status":"OK"}</pre>
|
||||
<code>{"status":"OK"}</code>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -17,20 +17,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
### Zeppelin Tutorial
|
||||
## Zeppelin Tutorial
|
||||
|
||||
We will assume you have Zeppelin installed already. If that's not the case, see [Install](../install/install.html).
|
||||
This tutorial walks you through some of the fundamental Zeppelin concepts. We will assume you have already installed Zeppelin. If not, please see [here](../install/install.html) first.
|
||||
|
||||
Zeppelin's current main backend processing engine is [Apache Spark](https://spark.apache.org). If you're new to the system, you might want to start by getting an idea of how it processes data to get the most out of Zeppelin.
|
||||
Current main backend processing engine of Zeppelin is [Apache Spark](https://spark.apache.org). If you're new to this system, you might want to start by getting an idea of how it processes data to get the most out of Zeppelin.
|
||||
|
||||
<br />
|
||||
### Tutorial with Local File
|
||||
## Tutorial with Local File
|
||||
|
||||
#### Data Refine
|
||||
### 1. Data Refine
|
||||
|
||||
Before you start Zeppelin tutorial, you will need to download [bank.zip](http://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip).
|
||||
|
||||
First, to transform data from csv format into RDD of `Bank` objects, run following script. This will also remove header using `filter` function.
|
||||
First, to transform csv format data into RDD of `Bank` objects, run following script. This will also remove header using `filter` function.
|
||||
|
||||
```scala
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ val bankText = sc.textFile("yourPath/bank/bank-full.csv")
|
|||
|
||||
case class Bank(age:Integer, job:String, marital : String, education : String, balance : Integer)
|
||||
|
||||
// split each line, filter out header (starts with "age"), and map it into Bank case class
|
||||
// split each line, filter out header (starts with "age"), and map it into Bank case class
|
||||
val bank = bankText.map(s=>s.split(";")).filter(s=>s(0)!="\"age\"").map(
|
||||
s=>Bank(s(0).toInt,
|
||||
s(1).replaceAll("\"", ""),
|
||||
|
|
@ -52,8 +52,7 @@ val bank = bankText.map(s=>s.split(";")).filter(s=>s(0)!="\"age\"").map(
|
|||
bank.toDF().registerTempTable("bank")
|
||||
```
|
||||
|
||||
<br />
|
||||
#### Data Retrieval
|
||||
### 2. Data Retrieval
|
||||
|
||||
Suppose we want to see age distribution from `bank`. To do this, run:
|
||||
|
||||
|
|
@ -74,9 +73,9 @@ Now we want to see age distribution with certain marital status and add combo bo
|
|||
```
|
||||
|
||||
<br />
|
||||
### Tutorial with Streaming Data
|
||||
## Tutorial with Streaming Data
|
||||
|
||||
#### Data Refine
|
||||
### 1. Data Refine
|
||||
|
||||
Since this tutorial is based on Twitter's sample tweet stream, you must configure authentication with a Twitter account. To do this, take a look at [Twitter Credential Setup](https://databricks-training.s3.amazonaws.com/realtime-processing-with-spark-streaming.html#twitter-credential-setup). After you get API keys, you should fill out credential related values(`apiKey`, `apiSecret`, `accessToken`, `accessTokenSecret`) with your API keys on following script.
|
||||
|
||||
|
|
@ -136,12 +135,11 @@ twt.print
|
|||
ssc.start()
|
||||
```
|
||||
|
||||
<br />
|
||||
#### Data Retrieval
|
||||
### 2. Data Retrieval
|
||||
|
||||
For each following script, every time you click run button you will see different result since it is based on real-time data.
|
||||
|
||||
Let's begin by extracting maximum 10 tweets which contain the word "girl".
|
||||
Let's begin by extracting maximum 10 tweets which contain the word **girl**.
|
||||
|
||||
```sql
|
||||
%sql select * from tweets where text like '%girl%' limit 10
|
||||
|
|
@ -154,7 +152,7 @@ This time suppose we want to see how many tweets have been created per sec durin
|
|||
```
|
||||
|
||||
|
||||
You can make user-defined function and use it in Spark SQL. Let's try it by making function named `sentiment`. This function will return one of the three attitudes(positive, negative, neutral) towards the parameter.
|
||||
You can make user-defined function and use it in Spark SQL. Let's try it by making function named `sentiment`. This function will return one of the three attitudes( positive, negative, neutral ) towards the parameter.
|
||||
|
||||
```scala
|
||||
def sentiment(s:String) : String = {
|
||||
|
|
|
|||
169
jdbc/pom.xml
Normal file
169
jdbc/pom.xml
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<version>0.6.0-incubating-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-jdbc</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>0.6.0-incubating-SNAPSHOT</version>
|
||||
<name>Zeppelin: JDBC interpreter</name>
|
||||
<url>http://www.apache.org</url>
|
||||
|
||||
<properties>
|
||||
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>${postgresql.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
<version>2.12.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<version>1.4.190</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-all</artifactId>
|
||||
<version>1.9.5</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.mockrunner</groupId>
|
||||
<artifactId>mockrunner-jdbc</artifactId>
|
||||
<version>1.0.8</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>2.7</version>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/jdbc</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>${project.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>${project.packaging}</type>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
413
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
Normal file
413
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
package org.apache.zeppelin.jdbc;
|
||||
|
||||
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
import org.apache.zeppelin.scheduler.SchedulerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.collect.Sets.SetView;
|
||||
|
||||
/**
|
||||
* JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ,
|
||||
* GreenplumDB, MariaDB, MySQL, Postgres and Redshit.
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code default.url} - JDBC URL to connect to.</li>
|
||||
* <li>{@code default.user} - JDBC user name..</li>
|
||||
* <li>{@code default.password} - JDBC password..</li>
|
||||
* <li>{@code default.driver.name} - JDBC driver name.</li>
|
||||
* <li>{@code common.max.result} - Max number of SQL result to display.</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>
|
||||
* How to use: <br/>
|
||||
* {@code %jdbc.sql} <br/>
|
||||
* {@code
|
||||
* SELECT store_id, count(*)
|
||||
* FROM retail_demo.order_lineitems_pxf
|
||||
* GROUP BY store_id;
|
||||
* }
|
||||
* </p>
|
||||
*
|
||||
*/
|
||||
public class JDBCInterpreter extends Interpreter {
|
||||
|
||||
private Logger logger = LoggerFactory.getLogger(JDBCInterpreter.class);
|
||||
|
||||
static final String COMMON_KEY = "common";
|
||||
static final String MAX_LINE_KEY = "max_count";
|
||||
static final String MAX_LINE_DEFAULT = "1000";
|
||||
|
||||
static final String DEFAULT_KEY = "default";
|
||||
static final String DRIVER_KEY = "driver";
|
||||
static final String URL_KEY = "url";
|
||||
static final String USER_KEY = "user";
|
||||
static final String PASSWORD_KEY = "password";
|
||||
static final String DOT = ".";
|
||||
|
||||
private static final char WHITESPACE = ' ';
|
||||
private static final char NEWLINE = '\n';
|
||||
private static final char TAB = '\t';
|
||||
private static final String TABLE_MAGIC_TAG = "%table ";
|
||||
private static final String EXPLAIN_PREDICATE = "EXPLAIN ";
|
||||
private static final String UPDATE_COUNT_HEADER = "Update Count";
|
||||
|
||||
static final String COMMON_MAX_LINE = COMMON_KEY + DOT + MAX_LINE_KEY;
|
||||
|
||||
static final String DEFAULT_DRIVER = DEFAULT_KEY + DOT + DRIVER_KEY;
|
||||
static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY;
|
||||
static final String DEFAULT_USER = DEFAULT_KEY + DOT + USER_KEY;
|
||||
static final String DEFAULT_PASSWORD = DEFAULT_KEY + DOT + PASSWORD_KEY;
|
||||
|
||||
static final String EMPTY_COLUMN_VALUE = "";
|
||||
|
||||
private final HashMap<String, Properties> propertiesMap;
|
||||
private final Map<String, Statement> paragraphIdStatementMap;
|
||||
|
||||
private final Map<String, ArrayList<Connection>> propertyKeyUnusedConnectionListMap;
|
||||
private final Map<String, Connection> paragraphIdConnectionMap;
|
||||
|
||||
static {
|
||||
Interpreter.register(
|
||||
"sql",
|
||||
"jdbc",
|
||||
JDBCInterpreter.class.getName(),
|
||||
new InterpreterPropertyBuilder()
|
||||
.add(DEFAULT_URL, "jdbc:postgresql://localhost:5432/", "The URL for JDBC.")
|
||||
.add(DEFAULT_USER, "gpadmin", "The JDBC user name")
|
||||
.add(DEFAULT_PASSWORD, "",
|
||||
"The JDBC user password")
|
||||
.add(DEFAULT_DRIVER, "org.postgresql.Driver", "JDBC Driver Name")
|
||||
.add(COMMON_MAX_LINE, MAX_LINE_DEFAULT,
|
||||
"Max number of SQL result to display.").build());
|
||||
}
|
||||
|
||||
public JDBCInterpreter(Properties property) {
|
||||
super(property);
|
||||
propertiesMap = new HashMap<>();
|
||||
propertyKeyUnusedConnectionListMap = new HashMap<>();
|
||||
paragraphIdStatementMap = new HashMap<>();
|
||||
paragraphIdConnectionMap = new HashMap<>();
|
||||
}
|
||||
|
||||
public HashMap<String, Properties> getPropertiesMap() {
|
||||
return propertiesMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open() {
|
||||
for (String propertyKey : property.stringPropertyNames()) {
|
||||
logger.debug("propertyKey: {}", propertyKey);
|
||||
String[] keyValue = propertyKey.split("\\.", 2);
|
||||
if (2 == keyValue.length) {
|
||||
logger.info("key: {}, value: {}", keyValue[0], keyValue[1]);
|
||||
Properties prefixProperties;
|
||||
if (propertiesMap.containsKey(keyValue[0])) {
|
||||
prefixProperties = propertiesMap.get(keyValue[0]);
|
||||
} else {
|
||||
prefixProperties = new Properties();
|
||||
propertiesMap.put(keyValue[0], prefixProperties);
|
||||
}
|
||||
prefixProperties.put(keyValue[1], property.getProperty(propertyKey));
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> removeKeySet = new HashSet<>();
|
||||
for (String key : propertiesMap.keySet()) {
|
||||
if (!COMMON_KEY.equals(key)) {
|
||||
Properties properties = propertiesMap.get(key);
|
||||
if (!properties.containsKey(DRIVER_KEY) || !properties.containsKey(URL_KEY)) {
|
||||
logger.error("{} will be ignored. {}.{} and {}.{} is mandatory.",
|
||||
key, DRIVER_KEY, key, key, URL_KEY);
|
||||
removeKeySet.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (String key : removeKeySet) {
|
||||
propertiesMap.remove(key);
|
||||
}
|
||||
|
||||
logger.debug("propertiesMap: {}", propertiesMap);
|
||||
}
|
||||
|
||||
public Connection getConnection(String propertyKey) throws ClassNotFoundException, SQLException {
|
||||
Connection connection = null;
|
||||
if (propertyKeyUnusedConnectionListMap.containsKey(propertyKey)) {
|
||||
ArrayList<Connection> connectionList = propertyKeyUnusedConnectionListMap.get(propertyKey);
|
||||
if (0 != connectionList.size()) {
|
||||
connection = propertyKeyUnusedConnectionListMap.get(propertyKey).remove(0);
|
||||
if (null != connection && connection.isClosed()) {
|
||||
connection.close();
|
||||
connection = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (null == connection) {
|
||||
Properties properties = propertiesMap.get(propertyKey);
|
||||
logger.info(properties.getProperty(DRIVER_KEY));
|
||||
Class.forName(properties.getProperty(DRIVER_KEY));
|
||||
String url = properties.getProperty(URL_KEY);
|
||||
String user = properties.getProperty(USER_KEY);
|
||||
String password = properties.getProperty(PASSWORD_KEY);
|
||||
if (null != user && null != password) {
|
||||
connection = DriverManager.getConnection(url, user, password);
|
||||
} else {
|
||||
connection = DriverManager.getConnection(url, properties);
|
||||
}
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
public Statement getStatement(String propertyKey, String paragraphId)
|
||||
throws SQLException, ClassNotFoundException {
|
||||
Connection connection;
|
||||
if (paragraphIdConnectionMap.containsKey(paragraphId)) {
|
||||
connection = paragraphIdConnectionMap.get(paragraphId);
|
||||
} else {
|
||||
connection = getConnection(propertyKey);
|
||||
}
|
||||
|
||||
Statement statement = connection.createStatement();
|
||||
if (isStatementClosed(statement)) {
|
||||
connection = getConnection(propertyKey);
|
||||
statement = connection.createStatement();
|
||||
}
|
||||
paragraphIdConnectionMap.put(paragraphId, connection);
|
||||
paragraphIdStatementMap.put(paragraphId, statement);
|
||||
|
||||
return statement;
|
||||
}
|
||||
|
||||
private boolean isStatementClosed(Statement statement) {
|
||||
try {
|
||||
return statement.isClosed();
|
||||
} catch (Throwable t) {
|
||||
logger.debug("{} doesn't support isClosed method", statement);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
try {
|
||||
for (List<Connection> connectionList : propertyKeyUnusedConnectionListMap.values()) {
|
||||
for (Connection c : connectionList) {
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
for (Statement statement : paragraphIdStatementMap.values()) {
|
||||
statement.close();
|
||||
}
|
||||
paragraphIdStatementMap.clear();
|
||||
|
||||
for (Connection connection : paragraphIdConnectionMap.values()) {
|
||||
connection.close();
|
||||
}
|
||||
paragraphIdConnectionMap.clear();
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.error("Error while closing...", e);
|
||||
}
|
||||
}
|
||||
|
||||
private InterpreterResult executeSql(String propertyKey, String sql,
|
||||
InterpreterContext interpreterContext) {
|
||||
|
||||
String paragraphId = interpreterContext.getParagraphId();
|
||||
|
||||
try {
|
||||
|
||||
Statement statement = getStatement(propertyKey, paragraphId);
|
||||
statement.setMaxRows(getMaxResult());
|
||||
|
||||
StringBuilder msg = null;
|
||||
boolean isTableType = false;
|
||||
|
||||
if (containsIgnoreCase(sql, EXPLAIN_PREDICATE)) {
|
||||
msg = new StringBuilder();
|
||||
} else {
|
||||
msg = new StringBuilder(TABLE_MAGIC_TAG);
|
||||
isTableType = true;
|
||||
}
|
||||
|
||||
ResultSet resultSet = null;
|
||||
try {
|
||||
|
||||
boolean isResultSetAvailable = statement.execute(sql);
|
||||
|
||||
if (isResultSetAvailable) {
|
||||
resultSet = statement.getResultSet();
|
||||
|
||||
ResultSetMetaData md = resultSet.getMetaData();
|
||||
|
||||
for (int i = 1; i < md.getColumnCount() + 1; i++) {
|
||||
if (i > 1) {
|
||||
msg.append(TAB);
|
||||
}
|
||||
msg.append(replaceReservedChars(isTableType, md.getColumnName(i)));
|
||||
}
|
||||
msg.append(NEWLINE);
|
||||
|
||||
int displayRowCount = 0;
|
||||
while (resultSet.next() && displayRowCount < getMaxResult()) {
|
||||
for (int i = 1; i < md.getColumnCount() + 1; i++) {
|
||||
msg.append(replaceReservedChars(isTableType, resultSet.getString(i)));
|
||||
if (i != md.getColumnCount()) {
|
||||
msg.append(TAB);
|
||||
}
|
||||
}
|
||||
msg.append(NEWLINE);
|
||||
displayRowCount++;
|
||||
}
|
||||
} else {
|
||||
// Response contains either an update count or there are no results.
|
||||
int updateCount = statement.getUpdateCount();
|
||||
msg.append(UPDATE_COUNT_HEADER).append(NEWLINE);
|
||||
msg.append(updateCount).append(NEWLINE);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
if (resultSet != null) {
|
||||
resultSet.close();
|
||||
}
|
||||
statement.close();
|
||||
} finally {
|
||||
statement = null;
|
||||
}
|
||||
}
|
||||
|
||||
return new InterpreterResult(Code.SUCCESS, msg.toString());
|
||||
|
||||
} catch (SQLException ex) {
|
||||
logger.error("Cannot run " + sql, ex);
|
||||
return new InterpreterResult(Code.ERROR, ex.getMessage());
|
||||
} catch (ClassNotFoundException e) {
|
||||
logger.error("Cannot run " + sql, e);
|
||||
return new InterpreterResult(Code.ERROR, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* For %table response replace Tab and Newline characters from the content.
|
||||
*/
|
||||
private String replaceReservedChars(boolean isTableResponseType, String str) {
|
||||
if (str == null) {
|
||||
return EMPTY_COLUMN_VALUE;
|
||||
}
|
||||
return (!isTableResponseType) ? str : str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
|
||||
logger.info("Run SQL command '{}'", cmd);
|
||||
String propertyKey = getPropertyKey(cmd);
|
||||
|
||||
if (null != propertyKey) {
|
||||
cmd = cmd.substring(propertyKey.length() + 2);
|
||||
} else {
|
||||
propertyKey = DEFAULT_KEY;
|
||||
}
|
||||
|
||||
cmd = cmd.trim();
|
||||
|
||||
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
|
||||
|
||||
return executeSql(propertyKey, cmd, contextInterpreter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
|
||||
logger.info("Cancel current query statement.");
|
||||
|
||||
String paragraphId = context.getParagraphId();
|
||||
try {
|
||||
paragraphIdStatementMap.get(paragraphId).cancel();
|
||||
} catch (SQLException e) {
|
||||
logger.error("Error while cancelling...", e);
|
||||
}
|
||||
}
|
||||
|
||||
public String getPropertyKey(String cmd) {
|
||||
int firstLineIndex = cmd.indexOf("\n");
|
||||
if (-1 == firstLineIndex) {
|
||||
firstLineIndex = cmd.length();
|
||||
}
|
||||
int configStartIndex = cmd.indexOf("(");
|
||||
int configLastIndex = cmd.indexOf(")");
|
||||
if (configStartIndex != -1 && configLastIndex != -1
|
||||
&& configLastIndex < firstLineIndex && configLastIndex < firstLineIndex) {
|
||||
return cmd.substring(configStartIndex + 1, configLastIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FormType getFormType() {
|
||||
return FormType.SIMPLE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getProgress(InterpreterContext context) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scheduler getScheduler() {
|
||||
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
|
||||
JDBCInterpreter.class.getName() + this.hashCode());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> completion(String buf, int cursor) {
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getMaxResult() {
|
||||
return Integer.valueOf(
|
||||
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
package org.apache.zeppelin.jdbc;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_KEY;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
|
||||
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.jdbc.JDBCInterpreter;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
|
||||
/**
|
||||
* JDBC interpreter unit tests
|
||||
*/
|
||||
public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
||||
|
||||
static String jdbcConnection;
|
||||
|
||||
private static String getJdbcConnection() throws IOException {
|
||||
if(null == jdbcConnection) {
|
||||
Path tmpDir = Files.createTempDirectory("h2-test-");
|
||||
tmpDir.toFile().deleteOnExit();
|
||||
jdbcConnection = format("jdbc:h2:%s", tmpDir);
|
||||
}
|
||||
return jdbcConnection;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
|
||||
Class.forName("org.h2.Driver");
|
||||
Connection connection = DriverManager.getConnection(getJdbcConnection());
|
||||
Statement statement = connection.createStatement();
|
||||
statement.execute(
|
||||
"DROP TABLE IF EXISTS test_table; " +
|
||||
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
|
||||
statement.execute(
|
||||
"insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name');"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaultProperties() throws SQLException {
|
||||
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(new Properties());
|
||||
|
||||
assertEquals("org.postgresql.Driver", jdbcInterpreter.getProperty(DEFAULT_DRIVER));
|
||||
assertEquals("jdbc:postgresql://localhost:5432/", jdbcInterpreter.getProperty(DEFAULT_URL));
|
||||
assertEquals("gpadmin", jdbcInterpreter.getProperty(DEFAULT_USER));
|
||||
assertEquals("", jdbcInterpreter.getProperty(DEFAULT_PASSWORD));
|
||||
assertEquals("1000", jdbcInterpreter.getProperty(COMMON_MAX_LINE));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSelectQuery() throws SQLException, IOException {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty("common.max_count", "1000");
|
||||
properties.setProperty("common.max_retry", "3");
|
||||
properties.setProperty("default.driver", "org.h2.Driver");
|
||||
properties.setProperty("default.url", getJdbcConnection());
|
||||
properties.setProperty("default.user", "");
|
||||
properties.setProperty("default.password", "");
|
||||
JDBCInterpreter t = new JDBCInterpreter(properties);
|
||||
t.open();
|
||||
|
||||
String sqlQuery = "select * from test_table";
|
||||
|
||||
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null));
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
|
||||
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSelectQueryMaxResult() throws SQLException, IOException {
|
||||
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty("common.max_count", "1");
|
||||
properties.setProperty("common.max_retry", "3");
|
||||
properties.setProperty("default.driver", "org.h2.Driver");
|
||||
properties.setProperty("default.url", getJdbcConnection());
|
||||
properties.setProperty("default.user", "");
|
||||
properties.setProperty("default.password", "");
|
||||
JDBCInterpreter t = new JDBCInterpreter(properties);
|
||||
t.open();
|
||||
|
||||
String sqlQuery = "select * from test_table";
|
||||
|
||||
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null));
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
|
||||
assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message());
|
||||
}
|
||||
}
|
||||
1
pom.xml
1
pom.xml
|
|
@ -94,6 +94,7 @@
|
|||
<module>hive</module>
|
||||
<module>phoenix</module>
|
||||
<module>postgresql</module>
|
||||
<module>jdbc</module>
|
||||
<module>tajo</module>
|
||||
<module>flink</module>
|
||||
<module>ignite</module>
|
||||
|
|
|
|||
|
|
@ -272,14 +272,12 @@
|
|||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-streaming_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-streaming-twitter_2.10</artifactId>
|
||||
|
|
@ -292,7 +290,6 @@
|
|||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- hadoop -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
|
|||
129
spark/pom.xml
129
spark/pom.xml
|
|
@ -38,64 +38,8 @@
|
|||
<spark.version>1.4.1</spark.version>
|
||||
<scala.version>2.10.4</scala.version>
|
||||
<scala.binary.version>2.10</scala.binary.version>
|
||||
<hadoop.version>2.3.0</hadoop.version>
|
||||
<py4j.version>0.8.2.1</py4j.version>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>vendor-repo</id>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>cloudera</id>
|
||||
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.1</id>
|
||||
<properties>
|
||||
<spark.version>1.1.1</spark.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.2</id>
|
||||
<properties>
|
||||
<spark.version>1.2.1</spark.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.3</id>
|
||||
<properties>
|
||||
<spark.version>1.3.1</spark.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.4</id>
|
||||
<properties>
|
||||
<spark.version>1.4.1</spark.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.5</id>
|
||||
<properties>
|
||||
<spark.version>1.5.2</spark.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>spark-1.6</id>
|
||||
<properties>
|
||||
<spark.version>1.6.0</spark.version>
|
||||
<py4j.version>0.9</py4j.version>
|
||||
</properties>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
|
@ -247,51 +191,6 @@
|
|||
<version>1.3</version>
|
||||
</dependency>
|
||||
|
||||
<!-- to compile and test code.
|
||||
Runtime dependency is provided by either spark-dependencies submodule or SPARK_HOME
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-repl_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-hive_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-catalyst_2.10</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
|
|
@ -326,12 +225,6 @@
|
|||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sf.py4j</groupId>
|
||||
<artifactId>py4j</artifactId>
|
||||
<version>${py4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!--TEST-->
|
||||
<dependency>
|
||||
|
|
@ -346,6 +239,28 @@
|
|||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.datanucleus</groupId>
|
||||
<artifactId>datanucleus-core</artifactId>
|
||||
<version>3.2.10</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.datanucleus</groupId>
|
||||
<artifactId>datanucleus-api-jdo</artifactId>
|
||||
<version>3.2.6</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.datanucleus</groupId>
|
||||
<artifactId>datanucleus-rdbms</artifactId>
|
||||
<version>3.2.9</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ limitations under the License.
|
|||
</img>
|
||||
|
||||
<div id="{{paragraph.id}}_error"
|
||||
class="error"
|
||||
class="error text"
|
||||
ng-if="paragraph.status == 'ERROR'"
|
||||
ng-bind="paragraph.errorMessage">
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -796,8 +796,10 @@ angular.module('zeppelinWebApp')
|
|||
$scope.moveUp();
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 74) { // Ctrl + Alt + j
|
||||
$scope.moveDown();
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 65) { // Ctrl + Alt + a
|
||||
$scope.insertNew('above');
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 66) { // Ctrl + Alt + b
|
||||
$scope.insertNew();
|
||||
$scope.insertNew('below');
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 79) { // Ctrl + Alt + o
|
||||
$scope.toggleOutput();
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 69) { // Ctrl + Alt + e
|
||||
|
|
|
|||
|
|
@ -78,6 +78,17 @@ limitations under the License.
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4">
|
||||
<div class="keys">
|
||||
<kbd class="kbd-dark">Ctrl</kbd> + <kbd class="kbd-dark">Alt</kbd> + <kbd class="kbd-dark">a</kbd>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-8">
|
||||
Insert new paragraph above
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-4">
|
||||
<div class="keys">
|
||||
|
|
|
|||
|
|
@ -402,6 +402,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
+ "org.apache.zeppelin.shell.ShellInterpreter,"
|
||||
+ "org.apache.zeppelin.hive.HiveInterpreter,"
|
||||
+ "org.apache.zeppelin.phoenix.PhoenixInterpreter,"
|
||||
+ "org.apache.zeppelin.postgresql.PostgreSqlInterpreter,"
|
||||
+ "org.apache.zeppelin.tajo.TajoInterpreter,"
|
||||
+ "org.apache.zeppelin.flink.FlinkInterpreter,"
|
||||
+ "org.apache.zeppelin.ignite.IgniteInterpreter,"
|
||||
|
|
@ -409,10 +410,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
+ "org.apache.zeppelin.lens.LensInterpreter,"
|
||||
+ "org.apache.zeppelin.cassandra.CassandraInterpreter,"
|
||||
+ "org.apache.zeppelin.geode.GeodeOqlInterpreter,"
|
||||
+ "org.apache.zeppelin.postgresql.PostgreSqlInterpreter,"
|
||||
+ "org.apache.zeppelin.kylin.KylinInterpreter,"
|
||||
+ "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,"
|
||||
+ "org.apache.zeppelin.scalding.ScaldingInterpreter"),
|
||||
+ "org.apache.zeppelin.scalding.ScaldingInterpreter,"
|
||||
+ "org.apache.zeppelin.jdbc.JDBCInterpreter"),
|
||||
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
|
||||
ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
|
||||
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
|
||||
|
|
|
|||
Loading…
Reference in a new issue