Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge from master

Browse source
This commit is contained in:
Rajat Venkatesh 2016-01-31 12:38:20 +05:30
commit 49b5b27777
150 changed files with 7803 additions and 4759 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
CONTRIBUTING.md
View file

@ -39,6 +39,29 @@ You can also use this small bookmarklet tool to fill your Pull Request fields au
javascript:(function() {var e = document.getElementById('pull_request_body');if (e) {e.value += '### What is this PR for?\nA few sentences describing the overall goals of the pull request\'s commits.\n\n### What type of PR is it?\n[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]\n\n### Todos\n* [ ] - Task\n\n### Is there a relevant Jira issue?\n\n### How should this be tested?\nOutline the steps to test the PR here.\n\n### Screenshots (if appropriate)\n\n### Questions:\n* Does the licenses files need update?\n* Is there breaking changes for older versions?\n* Does this needs documentation?';}})();
```
## Testing a Pull Request
You can also test and review a particular Pull Request. Here are two useful ways.
* Using a utility provided from Zeppelin.
```
dev/test_zeppelin_pr.py [# of PR]
```
For example, if you want to test `#513`, then the command will be:
```
dev/test_zeppelin_pr.py 513
```
* Another way is using [github/hub](https://github.com/github/hub).
```
hub checkout https://github.com/apache/incubator-zeppelin/pull/[# of PR]
```
The above two methods will help you test and review Pull Requests.
## Source Control Workflow
Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model.
@ -53,7 +76,6 @@ When a Pull Request is submitted, it is being merged or rejected by following re
* Committer can initiate lazy consensus ("Merge if there is no more discussion") and the code can be merged after certain time (normally 24 hours) when there is no review exists.
* Contributor can ping reviewers (including committer) by commenting 'Ready to review' or suitable indication.
## Becoming a Committer
The PPMC adds new committers from the active contributors, based on their contribution to Zeppelin. The qualifications for new committers include:

30
cassandra/pom.xml
View file

@ -35,18 +35,19 @@
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<cassandra.driver.version>2.1.7.1</cassandra.driver.version>
<cassandra.driver.version>3.0.0-rc1</cassandra.driver.version>
<snappy.version>1.0.5.4</snappy.version>
<lz4.version>1.2.0</lz4.version>
<scala.version>2.11.7</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<commons-lang.version>3.4</commons-lang.version>
<lz4.version>1.3.0</lz4.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<commons-lang.version>3.3.2</commons-lang.version>
<scalate.version>1.7.1</scalate.version>
<cassandra.guava.version>16.0.1</cassandra.guava.version>
<!--TEST-->
<scalatest.version>2.2.4</scalatest.version>
<junit.version>4.12</junit.version>
<achilles.version>3.2.2</achilles.version>
<achilles.version>3.2.4-Zeppelin</achilles.version>
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
</properties>
@ -65,6 +66,12 @@
<version>${cassandra.driver.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${cassandra.guava.version}</version>
</dependency>
<!-- Compression libraries for the cassandra-driver protocol. -->
<!-- Include both compression options to make to simplify deployment. -->
@ -130,7 +137,7 @@
<dependency>
<groupId>info.archinnov</groupId>
<artifactId>achilles-junit</artifactId>
<artifactId>achilles-embedded</artifactId>
<version>${achilles.version}</version>
<scope>test</scope>
<exclusions>
@ -145,13 +152,6 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@ -167,7 +167,6 @@
</dependency>
</dependencies>
<build>
<plugins>
<!-- Plugin to compile Scala code -->
@ -293,7 +292,6 @@
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

81
cassandra/src/main/resources/scalate/aggregateDetails.ssp Normal file
View file

@ -0,0 +1,81 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val sameNameAggregateDetails: SameNameAggregateDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
#for (aggregate <- sameNameAggregateDetails.aggregates)
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if(withCaption)
<caption><h4 class="text-success"><i class="glyphicon glyphicon-retweet"/>&nbsp;${aggregate.name}</h4></caption>
#end
<tbody>
<tr>
<td class="col-md-6"><strong>Keyspace</strong></td>
<td class="col-md-6 text-danger">${aggregate.keyspace}</td>
</tr>
<tr>
<td class="col-md-6"><strong>Arguments</strong></td>
<td class="col-md-6">${aggregate.arguments.toList.mkString(", ")}</td>
</tr>
<tr>
<td class="col-md-6"><strong>State Function</strong></td>
<td class="col-md-6">${aggregate.sFunc}</td>
</tr>
<tr>
<td class="col-md-6"><strong>State Type</strong></td>
<td class="col-md-6">${aggregate.sType}</td>
</tr>
#if(aggregate.finalFunc.isDefined)
<tr>
<td class="col-md-6"><strong>Final Function</strong></td>
<td class="col-md-6">${aggregate.finalFunc.get}</td>
</tr>
#end
#if(aggregate.initCond.isDefined)
<tr>
<td class="col-md-6"><strong>Initial State</strong></td>
<td class="col-md-6">${aggregate.initCond.get}</td>
</tr>
#end
<tr>
<td class="col-md-6"><strong>Return type</strong></td>
<td class="col-md-6">${aggregate.returnType}</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${aggregate.uniqueId}_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="text-success collapse" id="${aggregate.uniqueId}_asCQL">
<pre class="well">${aggregate.asCQL}</pre>
</div>
</div>
</div>
<hr/>
#end
</div>
<div class="col-md-2"></div>
</div>

68
cassandra/src/main/resources/scalate/allAggregates.ssp Normal file
View file

@ -0,0 +1,68 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(java.util.UUID)
<%@ val allAggregates: Map[(UUID, String), List[AggregateSummary]] %>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), aggregates) <- allAggregates)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${ksId}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksName}</span>
</a>
</h4>
</div>
<div id="${ksId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th>Aggregate</th>
<th>Return Type</th>
</tr>
</thead>
<tbody>
#for (aggregate <- aggregates)
<tr class="text-success">
<td>${aggregate.name + aggregate.arguments.mkString("(", ", ", ")")}</td>
<td>${aggregate.returnType}</td>
</tr>
#end
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
#end
</div>
</div>
</div>

68
cassandra/src/main/resources/scalate/allFunctions.ssp Normal file
View file

@ -0,0 +1,68 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(java.util.UUID)
<%@ val allFunctions: Map[(UUID, String), List[FunctionSummary]] %>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), functions) <- allFunctions)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${ksId}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksName}</span>
</a>
</h4>
</div>
<div id="${ksId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th>Function</th>
<th>Return Type</th>
</tr>
</thead>
<tbody>
#for (function <- functions)
<tr class="text-success">
<td>${function.name + function.arguments.mkString("(",", ", ")")}</td>
<td>${function.returnType}</td>
</tr>
#end
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
#end
</div>
</div>
</div>

68
cassandra/src/main/resources/scalate/allMaterializedViews.ssp Normal file
View file

@ -0,0 +1,68 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(java.util.UUID)
<%@ val allMVs: Map[(UUID,String),List[MaterializedViewSummary]] %>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), mvs) <- allMVs)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${ksId}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksName}</span>
</a>
</h4>
</div>
<div id="${ksId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>Materialized View</th></tr>
</thead>
<tbody>
#for (mv <- mvs)
<tr class="text-primary">
<td>
${mv.name}
&nbsp;<i class="glyphicon glyphicon-arrow-right"/>
&nbsp;<i class="glyphicon glyphicon-th-list"/>
&nbsp;${mv.baseTable}
</td>
</tr>
#end
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
#end
</div>
</div>
</div>

8
cassandra/src/main/resources/scalate/allTables.ssp
View file

@ -24,7 +24,6 @@
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), tables) <- allTables)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
@ -38,8 +37,6 @@
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
#if (tables.nonEmpty)
<table class="table">
<thead>
<tr><th>Tables</th></tr>
@ -52,10 +49,6 @@
</tbody>
</table>
#else
<span><h4>No Table</h4></span>
#end
</div>
<div class="col-md-2"/>
</div>
@ -63,7 +56,6 @@
</div>
</div>
#end
</div>
</div>
</div>

61
cassandra/src/main/resources/scalate/allUDTs.ssp Normal file
View file

@ -0,0 +1,61 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(java.util.UUID)
<%@ val allUDTs: Map[(UUID,String),List[String]] %>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), udts) <- allUDTs)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${ksId}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksName}</span>
</a>
</h4>
</div>
<div id="${ksId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>UDT</th></tr>
</thead>
<tbody>
#for (udt <- udts)
<tr class="text-warning"><td>${udt}</td></tr>
#end
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
#end
</div>
</div>
</div>

37
cassandra/src/main/resources/scalate/dropDownMenuForKeyspace.ssp
View file

@ -35,8 +35,19 @@
#end
#end
#if(ksContent.udts.nonEmpty)
#if(ksContent.views.nonEmpty)
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-primary">Materialized Views</span></li>
#for((id,name,_) <- ksContent.views)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-primary"><i class="glyphicon glyphicon-eye-open"/>&nbsp;${name}</span>
</a>
</li>
#end
#end
#if(ksContent.udts.nonEmpty)
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-warning">User Defined Types</span></li>
#for((id,name,_) <- ksContent.udts)
@ -47,6 +58,30 @@
</li>
#end
#end
#if(ksContent.functions.nonEmpty)
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-success">Functions</span></li>
#for((id,name,_) <- ksContent.functions)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-success"><i class="glyphicon glyphicon-random"/>&nbsp;${name}</span>
</a>
</li>
#end
#end
#if(ksContent.aggregates.nonEmpty)
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-success">Aggregates</span></li>
#for((id,name,_) <- ksContent.aggregates)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-success"><i class="glyphicon glyphicon-retweet"/>&nbsp;${name}</span>
</a>
</li>
#end
#end
</ul>
</a>
</li>

77
cassandra/src/main/resources/scalate/functionDetails.ssp Normal file
View file

@ -0,0 +1,77 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val sameNameFunctionDetails: SameNameFunctionDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
#for (function <- sameNameFunctionDetails.functions)
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if(withCaption)
<caption><h4 class="text-success"><i class="glyphicon glyphicon-random"/>&nbsp;${function.name}</h4></caption>
#end
<tbody>
<tr>
<td class="col-md-6"><strong>Keyspace</strong></td>
<td class="col-md-6 text-danger">${function.keyspace}</td>
</tr>
<tr>
<td class="col-md-6"><strong>Arguments</strong></td>
<td class="col-md-6">${function.arguments.toList.mkString(", ")}</td>
</tr>
<tr>
<td class="col-md-6"><strong>Null Input</strong></td>
#if(function.calledOnNullInput)
<td class="col-md-6">CALLED ON NULL INPUT</td>
#else
<td class="col-md-6">RETURN NULL ON NULL INPUT</td>
#end
</tr>
<tr>
<td class="col-md-6"><strong>Return type</strong></td>
<td class="col-md-6">${function.returnType}</td>
</tr>
<tr>
<td class="col-md-6"><strong>Language</strong></td>
<td class="col-md-6">${function.language}</td>
</tr>
<tr>
<td class="col-md-6"><strong>Body</strong></td>
<td class="col-md-6">${escape(function.body)}</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${function.uniqueId}_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="text-success collapse" id="${function.uniqueId}_asCQL">
<pre class="well">${function.asCQL}</pre>
</div>
</div>
</div>
<hr/>
#end
</div>
<div class="col-md-2"></div>
</div>

135
cassandra/src/main/resources/scalate/helpMenu.ssp
View file

@ -27,7 +27,9 @@
<%@ val preparedStatementsId: UUID = UUIDs.random() %>
<%@ val dynamicFormsId: UUID = UUIDs.random() %>
<%@ val configurationId: UUID = UUIDs.random() %>
<%@ val miscId: UUID = UUIDs.random() %>
<%@ val sharedStatesId: UUID = UUIDs.random() %>
<%@ val changelogId: UUID = UUIDs.random() %>
<%@ val contactsId: UUID = UUIDs.random() %>
<br/>
<br/>
@ -70,10 +72,20 @@
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${miscId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Misc</span>
<a role="button" data-toggle="collapse" data-target="#${sharedStatesId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Shared States</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${changelogId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Change Log</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${contactsId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Contacts &amp; Bugs</span>
</a>
</li>
</ul>
</a>
</li>
@ -91,12 +103,12 @@
<ul class="dropdown-menu">
<li>
<a role="button">
<span class="text-info">Version <strong>1.0</strong></span>
<span class="text-info">Version <strong>2.0</strong></span>
</a>
</li>
<li>
<a role="button">
<span class="text-info">Java Driver Version <strong>2.1.7.1</strong></span>
<span class="text-info">Java Driver Version <strong>3.0.0-rc1</strong></span>
</a>
</li>
<li>
@ -237,7 +249,7 @@
<h3>II Comments</h3>
<p>
It is possible to add comments between statements. Single line comments start with the
<strong>hash</strong> sign (#). Multi-line comments are enclosed between
<strong>hash</strong> sign (#) or <strong>double slashes</strong> (//). Multi-line comments are enclosed between
<strong>&sol;&ast;&ast;</strong> and <strong>&ast;&ast;&sol;</strong>. Ex:
<br/>
@ -246,9 +258,12 @@
<div class="col-md-8 col-md-offset-2">
<pre>
#First comment
#Single line comment style 1
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
//Single line comment style 2
/**
Multi line
comments
@ -313,6 +328,22 @@
<td><strong>DESCRIBE TABLES;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the tables name</td>
</tr>
<tr>
<td><strong>DESCRIBE TYPES;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the types name</td>
</tr>
<tr>
<td><strong>DESCRIBE FUNCTIONS;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the functions name and arguments</td>
</tr>
<tr>
<td><strong>DESCRIBE AGGREGATES;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the aggregates name and arguments</td>
</tr>
<tr>
<td><strong>DESCRIBE MATERIALIZED VIEWS;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the materialized view name</td>
</tr>
<tr>
<td><strong>DESCRIBE KEYSPACE &lt;keyspace name&gt;;</strong></td>
<td>Describe the given keyspace configuration and all its table details (name, columns, ...)</td>
@ -333,6 +364,30 @@
the default <em>system</em> keyspace is used. If no type is found, an error message is raised
</td>
</tr>
<tr>
<td><strong>DESCRIBE FUNCTION <em>(&lt;keyspace name&gt;).</em>&lt;function name&gt;;</strong></td>
<td>
Describe the given function. If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no function is found, an error message is raised
</td>
</tr>
<tr>
<td><strong>DESCRIBE AGGREGATE <em>(&lt;keyspace name&gt;).</em>&lt;aggregate name&gt;;</strong></td>
<td>
Describe the given aggregate. If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no aggregate is found, an error message is raised
</td>
</tr>
<tr>
<td><strong>DESCRIBE MATERIALIZED VIEW <em>(&lt;keyspace name&gt;).</em>&lt;view name&gt;;</strong></td>
<td>
Describe the given materialized view. If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no materialized view is found, an error message is raised
</td>
</tr>
</tbody>
</table>
<br/>
@ -342,9 +397,7 @@
</p>
<h3>II Schema Display</h3>
<p>
The schema objects (cluster, keyspace, table &amp; type) are displayed in a tabular format.
There is a <strong>drop-down</strong> menu on the top left corner to expand objects details.
On the top right menu is shown the Icon legend.
The schema objects (cluster, keyspace, table, type, view, function &amp; aggregate) are displayed in a tabular format. There is a <strong>drop-down</strong> menu on the top left corner to expand objects details. On the top right menu is shown the Icon legend.
</p>
</div>
@ -886,16 +939,68 @@
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${miscId}" aria-expanded="false">
<span class="text-info"><strong>Miscellaneous</strong></span>
<a role="button" data-toggle="collapse" data-target="#${sharedStatesId}" aria-expanded="false">
<span class="text-info"><strong>Shared states</strong></span>
</a>
</h4>
</div>
<div id="${miscId}" class="panel-collapse collapse" role="tabpanel">
<div id="${sharedStatesId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<h3>Execution parallelism</h3>
It is possible to execute many paragraphs in parallel. However, at the back-end side, we’re still using <strong>synchronous</strong> queries. Asynchronous execution is only possible when it is possible to return a Future value in the <strong>InterpreterResult</strong>. It may be an interesting proposal for the Zeppelin project.
It is possible to execute many paragraphs in parallel. However, at the back-end side, we’re still using synchronous queries. <em>Asynchronous execution</em> is only possible when it is possible to return a <strong>Future</strong> value in the <strong>InterpreterResult</strong>. It may be an interesting proposal for the <strong>Zeppelin</strong> project.
<br/>
Another caveat is that the same <strong>com.datastax.driver.core.Session</strong> object is used for <strong>all</strong> notebooks and paragraphs. Consequently, if you use the <em>USE keyspace name;</em> statement to log into a keyspace, it will change the keyspace for <strong>all current users</strong> of the Cassandra interpreter because we only create 1 <strong>com.datastax.driver.core.Session</strong> object per instance of <strong>Cassandra</strong> interpreter.
<br/>
The same remark does apply to the <strong>prepared statement hash map</strong>, it is shared by <strong>all users</strong> using the same instance of <strong>Cassandra</strong> interpreter.
<br/>
Until <strong>Zeppelin</strong> offers a real multi-users separation, there is a work-around to segregate user environment and states: <em>create different Cassandra interpreter instances</em>
<br/>
<ol>
<li>First go to the <strong>Interpreter</strong> menu and click on the <strong>Create</strong> button</li>
<li>In the interpreter creation form, put <strong>cass-instance2</strong> as <strong>Name</strong> and select the <strong>cassandra</strong> in the interpreter drop-down list</li>
<li>Click on <strong>Save</strong> to create the new interpreter instance. Now you should be able to see it in the interpreter list</li>
<li>Go back to your notebook and click on the <strong>Gear</strong> icon to configure interpreter bindings. You should be able to see and select the <strong>cass-instance2</strong> interpreter instance in the available interpreter list instead of the standard <strong>cassandra</strong> instance</li>
</ol>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${changelogId}" aria-expanded="false">
<span class="text-info"><strong>Change Log</strong></span>
</a>
</h4>
</div>
<div id="${changelogId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<strong>2.0</strong>&nbsp;:
<br/>
<ul>
<li>Update help menu and add changelog</li>
<li>Add Support for User Defined Functions, User Defined Aggregates and Materialized Views</li>
<li>Upgrade Java driver version to <strong>3.0.0-rc1</strong></li>
</ul>
<strong>1.0</strong>&nbsp;:
<br/>
<ul>
<li>Initial version</li>
</ul>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${contactsId}" aria-expanded="false">
<span class="text-info"><strong>Contact &amp; Bugs</strong></span>
</a>
</h4>
</div>
<div id="${contactsId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
If you encounter a bug for this interpreter, please create a <a href="https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN" target="_blank"><strong>JIRA</strong></a> ticket and ping me on Twitter at <a href="https://twitter.com/doanduyhai" target="_blank"><strong>@doanduyhai</strong></a>
</div>
</div>
</div>
</div>

90
cassandra/src/main/resources/scalate/keyspaceContent.ssp
View file

@ -57,6 +57,36 @@
</div>
#end
#if (ksContent.views.nonEmpty)
<!-- Materialized Views -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-primary">Materialized Views</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for((id,name,viewHTML) <- ksContent.views)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-primary"><i class="glyphicon glyphicon-eye-open"/>&nbsp;${name}</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(viewHTML)}
</div>
</div>
</div>
#end
</div>
</div>
#end
#if (ksContent.udts.nonEmpty)
<!-- UDTs -->
<table width="100%">
@ -88,4 +118,64 @@
</div>
#end
#if (ksContent.functions.nonEmpty)
<!-- Functions -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-success">Functions</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for((id,name,functionHTML) <- ksContent.functions)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-success"><i class="glyphicon glyphicon-random"/>&nbsp;${name}</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(functionHTML)}
</div>
</div>
</div>
#end
</div>
</div>
#end
#if (ksContent.aggregates.nonEmpty)
<!-- Aggregates -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-success">Aggregates</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for((id,name,aggregateHTML) <- ksContent.aggregates)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-success"><i class="glyphicon glyphicon-retweet"/>&nbsp;${name}</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(aggregateHTML)}
</div>
</div>
</div>
#end
</div>
</div>
#end
</div>

2
cassandra/src/main/resources/scalate/keyspaceDetails.ssp
View file

@ -46,7 +46,7 @@
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${ksDetails.uniqueId}_asCQL">
<a data-toggle="collapse" data-target="#${ksDetails.uniqueId}_asCQL" class="text-danger">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>

109
cassandra/src/main/resources/scalate/materializedViewDetails.ssp Normal file
View file

@ -0,0 +1,109 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val mvDetails: MaterializedViewDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if(withCaption)
<caption>
<h4 class="text-primary">
<i class="glyphicon glyphicon-eye-open"/>
&nbsp;${mvDetails.name}
&nbsp;<i class="glyphicon glyphicon-arrow-right"/>
&nbsp;<i class="glyphicon glyphicon-th-list"/>
&nbsp;${mvDetails.baseTable}&nbsp;
</h4>
</caption>
#end
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
#for (column <- mvDetails.columns)
#match (column.columnType)
#case(PartitionKey)
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(StaticColumn)
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(ClusteringColumn(ASC))
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes" title="Sort ASC"/>
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(ClusteringColumn(DESC))
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#otherwise
<tr>
<td class="col-md-4"></td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#end
#end
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${mvDetails.uniqueId}_asCQL" class="text-primary">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="${mvDetails.uniqueId}_asCQL">
<pre class="well">${mvDetails.asCQL}</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>

22
cassandra/src/main/resources/scalate/menu.ssp
View file

@ -54,6 +54,23 @@
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-eye-open text-primary" />&nbsp;&nbsp;Materialized View
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-random text-success" />&nbsp;&nbsp;Function
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-retweet text-success" />&nbsp;&nbsp;Aggregate
</a>
</li>
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-primary">Table icons</span></li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
@ -79,11 +96,6 @@
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>

83
cassandra/src/main/resources/scalate/tableDetails.ssp
View file

@ -42,15 +42,6 @@
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
@ -59,15 +50,6 @@
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
@ -78,15 +60,6 @@
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes" title="Sort ASC"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
@ -97,32 +70,13 @@
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#otherwise
<tr>
<td class="col-md-4">
#match (column.index)
#case (Some(index))
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4"></td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
@ -132,7 +86,7 @@
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${tableDetails.uniqueId}_asCQL">
<a data-toggle="collapse" data-target="#${tableDetails.uniqueId}_asCQL" class="text-primary">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
@ -142,6 +96,39 @@
</div>
</div>
</div>
#if(tableDetails.indices.size > 0)
<hr/>
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-danger"><i class="glyphicon glyphicon-info-sign"/>&nbsp;${tableDetails.tableName}'s indices</h4>
</caption>
<thead>
<tr>
<th class="col-md-6">Name</th>
<th class="col-md-6">Target</th>
</tr>
</thead>
<tbody>
#for (index <- tableDetails.indices)
<tr>
<td class="col-md-6">${index.name}</td>
<td class="col-md-6">${index.target}</td>
</tr>
#end
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${tableDetails.uniqueId}_indices_asCQL" class="text-danger">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="${tableDetails.uniqueId}_indices_asCQL">
<pre class="well">${tableDetails.indicesAsCQL}</pre>
</div>
</div>
</div>
#end
</div>
<div class="col-md-2"></div>
</div>

2
cassandra/src/main/resources/scalate/udtDetails.ssp
View file

@ -46,7 +46,7 @@
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${udtDetails.uniqueId}_asCQL">
<a data-toggle="collapse" data-target="#${udtDetails.uniqueId}_asCQL" class="text-warning">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>

23
cassandra/src/main/scala/com/datastax/driver/core/TableMetadataWrapper.scala Normal file
View file

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.datastax.driver.core
case class TableMetadataWrapper(val meta: TableMetadata) {
def exportTableOnlyAsString(): String = {
meta.asCQLQuery(true)
}
}

417
cassandra/src/main/scala/org/apache/zeppelin/cassandra/DisplaySystem.scala
View file

@ -18,7 +18,6 @@ package org.apache.zeppelin.cassandra
import java.util.UUID
import com.datastax.driver.core.ColumnMetadata.IndexMetadata
import com.datastax.driver.core.utils.UUIDs
import org.apache.zeppelin.cassandra.MetaDataHierarchy._
import org.fusesource.scalate.TemplateEngine
@ -39,16 +38,30 @@ object DisplaySystem {
val CLUSTER_DETAILS_TEMPLATE = "scalate/clusterDetails.ssp"
val KEYSPACE_DETAILS_TEMPLATE = "scalate/keyspaceDetails.ssp"
val TABLE_DETAILS_TEMPLATE = "scalate/tableDetails.ssp"
val ALL_TABLES_TEMPLATE = "scalate/allTables.ssp"
val UDT_DETAILS_TEMPLATE = "scalate/udtDetails.ssp"
val ALL_UDTS_TEMPLATE = "scalate/allUDTs.ssp"
val FUNCTION_DETAILS_TEMPLATE = "scalate/functionDetails.ssp"
val ALL_FUNCTIONS_TEMPLATE = "scalate/allFunctions.ssp"
val AGGREGATE_DETAILS_TEMPLATE = "scalate/aggregateDetails.ssp"
val ALL_AGGREGATES_TEMPLATE = "scalate/allAggregates.ssp"
val MATERIALIZED_VIEW_DETAILS_TEMPLATE = "scalate/materializedViewDetails.ssp"
val ALL_MATERIALIZED_VIEWS_TEMPLATE = "scalate/allMaterializedViews.ssp"
val MENU_TEMPLATE = "scalate/menu.ssp"
val CLUSTER_DROPDOWN_TEMPLATE = "scalate/dropDownMenuForCluster.ssp"
val KEYSPACE_DROPDOWN_TEMPLATE = "scalate/dropDownMenuForKeyspace.ssp"
val KEYSPACE_DROPDOWN_TEMPLATE = "scalate/dropDownMenuForKeyspace.ssp"
val CLUSTER_CONTENT_TEMPLATE = "scalate/clusterContent.ssp"
val KEYSPACE_CONTENT_TEMPLATE = "scalate/keyspaceContent.ssp"
val ALL_TABLES_TEMPLATE = "scalate/allTables.ssp"
object TableDisplay {
@ -58,10 +71,12 @@ object DisplaySystem {
protected[DisplaySystem] def formatWithoutMenu(meta: TableMetadata, withCaption: Boolean): String = {
val tableName: String = meta.getName
val columnsDetails = MetaDataConverter.tableMetaToColumnDetails(meta)
val columnsDetails = MetaDataConverter.Table.tableMetaToColumnDetails(meta)
val indicesDetails = MetaDataConverter.Table.tableMetaToIndexDetails(meta)
val indicesAsCQL = indicesDetails.map(_.asCQL).mkString("\n")
engine.layout(TABLE_DETAILS_TEMPLATE,
Map[String, Any]("tableDetails" -> TableDetails(tableName, columnsDetails, meta.exportAsString), "withCaption" -> withCaption))
Map[String, Any]("tableDetails" -> TableDetails(tableName, columnsDetails, indicesDetails, TableMetadataWrapper(meta).exportTableOnlyAsString(), indicesAsCQL), "withCaption" -> withCaption))
}
}
@ -72,13 +87,49 @@ object DisplaySystem {
protected[DisplaySystem] def formatWithoutMenu(userType: UserType, withCaption: Boolean): String = {
val udtName: String = userType.getTypeName
val columnsDetails = MetaDataConverter.userTypeToColumnDetails(userType)
val columnsDetails = MetaDataConverter.UDT.userTypeToColumnDetails(userType)
engine.layout(UDT_DETAILS_TEMPLATE,
Map[String, Any]("udtDetails" -> UDTDetails(udtName, columnsDetails, userType.exportAsString), "withCaption" -> withCaption))
Map[String, Any]("udtDetails" -> UDTDetails(udtName,columnsDetails,userType.exportAsString()), "withCaption" -> withCaption))
}
}
object FunctionDisplay {
def format(statement: String, functions: List[FunctionMetadata], withCaption: Boolean): String = {
MenuDisplay.formatMenu(statement) ++ formatWithoutMenu(functions, withCaption)
}
protected[DisplaySystem] def formatWithoutMenu(functions: List[FunctionMetadata], withCaption: Boolean): String = {
val functionDetails: List[FunctionDetails] = functions.map(MetaDataConverter.functionMetaToFunctionDetails(_))
engine.layout(FUNCTION_DETAILS_TEMPLATE,
Map[String, Any]("sameNameFunctionDetails" -> SameNameFunctionDetails(functionDetails), "withCaption" -> withCaption))
}
}
object AggregateDisplay {
def format(statement: String, aggregates: List[AggregateMetadata], withCaption: Boolean, codecRegistry: CodecRegistry): String = {
MenuDisplay.formatMenu(statement) ++ formatWithoutMenu(aggregates, withCaption, codecRegistry)
}
protected[DisplaySystem] def formatWithoutMenu(aggregates: List[AggregateMetadata], withCaption: Boolean, codecRegistry: CodecRegistry): String = {
val aggDetails: List[AggregateDetails] = aggregates.map(agg => MetaDataConverter.aggregateMetaToAggregateDetails(codecRegistry, agg))
engine.layout(AGGREGATE_DETAILS_TEMPLATE,
Map[String, Any]("sameNameAggregateDetails" -> SameNameAggregateDetails(aggDetails), "withCaption" -> withCaption))
}
}
object MaterializedViewDisplay {
def format(statement: String, mv: MaterializedViewMetadata, withCaption: Boolean): String = {
MenuDisplay.formatMenu(statement) ++ formatWithoutMenu(mv, withCaption)
}
protected[DisplaySystem] def formatWithoutMenu(mv: MaterializedViewMetadata, withCaption: Boolean): String = {
val mvDetails = MetaDataConverter.mvMetaToMaterializedViewDetails(mv)
engine.layout(MATERIALIZED_VIEW_DETAILS_TEMPLATE,
Map[String, Any]("mvDetails" -> mvDetails, "withCaption" -> withCaption))
}
}
object KeyspaceDisplay {
private def formatCQLQuery(cql: String): String = {
@ -97,19 +148,32 @@ object DisplaySystem {
Map[String, Any]("ksDetails" -> ksDetails, "withCaption" -> withCaption))
}
def formatKeyspaceContent(statement: String, meta: KeyspaceMetadata): String = {
def formatKeyspaceContent(statement: String, meta: KeyspaceMetadata, codecRegistry: CodecRegistry): String = {
val ksName: String = meta.getName
val ksDetails = formatKeyspaceOnly(meta, false)
val ksDetails = formatKeyspaceOnly(meta, true)
val tableDetails: List[(UUID, String, String)] = meta.getTables.asScala.toList
.sortBy(meta => meta.getName)
.map(meta => (UUIDs.timeBased(), meta.getName, TableDisplay.formatWithoutMenu(meta, false)))
.sortBy(_.getName)
.map(table => (UUIDs.timeBased(), table.getName, TableDisplay.formatWithoutMenu(table, false)))
val viewDetails: List[(UUID, String, String)] = meta.getMaterializedViews.asScala.toList
.sortBy(_.getName)
.map(view => (UUIDs.timeBased(), view.getName, MaterializedViewDisplay.formatWithoutMenu(view, false)))
val udtDetails: List[(UUID, String, String)] = meta.getUserTypes.asScala.toList
.sortBy(udt => udt.getTypeName)
.sortBy(_.getTypeName)
.map(udt => (UUIDs.timeBased(), udt.getTypeName, UDTDisplay.formatWithoutMenu(udt, false)))
val ksContent: KeyspaceContent = KeyspaceContent(ksName, ksDetails, tableDetails, udtDetails)
val functionDetails: List[(UUID, String, String)] = meta.getFunctions.asScala.toList
.sortBy(_.getSimpleName)
.map(function => (UUIDs.timeBased(), function.getSimpleName, FunctionDisplay.formatWithoutMenu(List(function), false)))
val aggregateDetails: List[(UUID, String, String)] = meta.getAggregates.asScala.toList
.sortBy(_.getSimpleName)
.map(agg => (UUIDs.timeBased(), agg.getSimpleName, AggregateDisplay.formatWithoutMenu(List(agg), false, codecRegistry)))
val ksContent: KeyspaceContent = KeyspaceContent(ksName, ksDetails, tableDetails, viewDetails,
udtDetails, functionDetails, aggregateDetails)
MenuDisplay.formatMenuForKeyspace(statement, ksContent) +
engine.layout(KEYSPACE_CONTENT_TEMPLATE,
@ -144,26 +208,152 @@ object DisplaySystem {
def formatAllTables(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.filter(_.getTables.size > 0)
.sortBy(ks => ks.getName)
if(ksMetas.isEmpty) {
NoResultDisplay.formatNoResult
} else {
val allTables: Map[(UUID, String), List[String]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getTables.asScala.toList.map(table => table.getName).sortBy(name => name))
})
.sortBy{case ((id,name), _) => name}
val keyspaceDetails: List[(UUID, String, String)] = allTables
.keySet.toList.sortBy{case(id,ksName) => ksName}
.map{case(id,ksName) => (id,ksName, "")}
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_TABLES_TEMPLATE,
Map[String, Any]("allTables" -> allTables))
}
}
def formatAllUDTs(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.filter(_.getUserTypes.size > 0)
.sortBy(ks => ks.getName)
val allTables: Map[(UUID, String), List[String]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getTables.asScala.toList.map(table => table.getName).sortBy(name => name))
})
.sortBy{case ((id,name), _) => name}
if(ksMetas.isEmpty) {
NoResultDisplay.formatNoResult
} else {
val allUDTs: Map[(UUID, String), List[String]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getUserTypes.asScala.toList.map(udt => udt.getTypeName).sortBy(name => name))
})
.sortBy { case ((id, name), _) => name }
val keyspaceDetails: List[(UUID, String, String)] = allTables
.keySet.toList.sortBy{case(id,ksName) => ksName}
.map{case(id,ksName) => (id,ksName, "")}
val keyspaceDetails: List[(UUID, String, String)] = allUDTs
.keySet.toList.sortBy { case (id, ksName) => ksName }
.map { case (id, ksName) => (id, ksName, "") }
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_TABLES_TEMPLATE,
Map[String, Any]("allTables" -> allTables))
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_UDTS_TEMPLATE,
Map[String, Any]("allUDTs" -> allUDTs))
}
}
def formatAllFunctions(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.filter(_.getFunctions.size > 0)
.sortBy(ks => ks.getName)
if(ksMetas.isEmpty) {
NoResultDisplay.formatNoResult
} else {
val allFunctions: Map[(UUID, String), List[FunctionSummary]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getFunctions.asScala.toList
.map(MetaDataConverter.functionMetaToFunctionSummary(_))
.sortBy(_.name))
})
.sortBy { case ((id, name), _) => name }
val keyspaceDetails: List[(UUID, String, String)] = allFunctions
.keySet.toList.sortBy { case (id, ksName) => ksName }
.map { case (id, ksName) => (id, ksName, "") }
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_FUNCTIONS_TEMPLATE,
Map[String, Any]("allFunctions" -> allFunctions))
}
}
def formatAllAggregates(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.filter(_.getAggregates.size > 0)
.sortBy(ks => ks.getName)
if(ksMetas.isEmpty) {
NoResultDisplay.formatNoResult
} else {
val allAggregates: Map[(UUID, String), List[AggregateSummary]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getAggregates.asScala.toList
.map(MetaDataConverter.aggregateMetaToAggregateSummary(_))
.sortBy(_.name))
})
.sortBy { case ((id, name), _) => name }
val keyspaceDetails: List[(UUID, String, String)] = allAggregates
.keySet.toList.sortBy { case (id, ksName) => ksName }
.map { case (id, ksName) => (id, ksName, "") }
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_AGGREGATES_TEMPLATE,
Map[String, Any]("allAggregates" -> allAggregates))
}
}
def formatAllMaterializedViews(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.filter(_.getMaterializedViews.size > 0)
.sortBy(ks => ks.getName)
if(ksMetas.isEmpty) {
NoResultDisplay.formatNoResult
} else {
val allMVs: Map[(UUID, String), List[MaterializedViewSummary]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getMaterializedViews.asScala.toList
.map(MetaDataConverter.mvMetaToMaterializedViewSummary(_))
.sortBy(_.name))
})
.sortBy { case ((id, name), _) => name }
val keyspaceDetails: List[(UUID, String, String)] = allMVs
.keySet.toList.sortBy { case (id, ksName) => ksName }
.map { case (id, ksName) => (id, ksName, "") }
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_MATERIALIZED_VIEWS_TEMPLATE,
Map[String, Any]("allMVs" -> allMVs))
}
}
}
@ -233,63 +423,150 @@ class ColumnMetaWrapper(val columnMeta: ColumnMetadata) {
*/
object MetaDataConverter {
def tableMetaToColumnDetails(meta: TableMetadata): List[ColumnDetails] = {
val partitionKeys: List[ColumnMetaWrapper] = meta.getPartitionKey.asScala.toList.map(new ColumnMetaWrapper(_))
val clusteringColumns: List[ColumnMetaWrapper] = meta.getClusteringColumns.asScala.toList.map(new ColumnMetaWrapper(_))
val columns: List[ColumnMetaWrapper] = meta.getColumns.asScala.toList.map(new ColumnMetaWrapper(_))
.diff(partitionKeys).diff(clusteringColumns)
val clusteringOrders = meta.getClusteringOrder.asScala.toList
type DriverClusteringOrder = com.datastax.driver.core.ClusteringOrder
convertPartitionKeys(partitionKeys):::
extractStaticColumns(columns):::
convertClusteringColumns(clusteringColumns, clusteringOrders):::
extractNormalColumns(columns)
}
def userTypeToColumnDetails(userType: UserType): List[ColumnDetails] = {
userType.getFieldNames.asScala.toList
.map(name => new ColumnDetails(name, NormalColumn, userType.getFieldType(name), None))
def functionMetaToFunctionDetails(function: FunctionMetadata): FunctionDetails = {
new FunctionDetails(function.getKeyspace.getName,
function.getSimpleName,
function.getArguments.asScala
.toMap
.map{case(paramName, dataType) => paramName + " " + dataType.asFunctionParameterString()}
.toList,
function.isCalledOnNullInput,
function.getReturnType.asFunctionParameterString(),
function.getLanguage,
function.getBody,
function.exportAsString())
}
private def extractNormalColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == false)
.map(c => new ColumnDetails(c.columnMeta.getName, NormalColumn, c.columnMeta.getType, extractIndexDetail(c)))
def functionMetaToFunctionSummary(function: FunctionMetadata): FunctionSummary = {
new FunctionSummary(function.getKeyspace.getName,
function.getSimpleName,
function.getArguments.asScala.toMap
.mapValues(dataType => dataType.asFunctionParameterString())
.values.toList,
function.getReturnType.asFunctionParameterString()
)
}
private def extractIndexDetail(column: ColumnMetaWrapper): Option[IndexDetails] = {
val indexOption = Option(column.columnMeta.getIndex)
def buildIndexInfo(indexMeta: IndexMetadata): String = {
if(indexMeta.isKeys) "KEYS"
if(indexMeta.isEntries) "ENTRIES"
if(indexMeta.isFull) "FULL"
if(indexMeta.isCustomIndex) s"Class = ${indexMeta.getIndexClassName}"
else ""
def aggregateMetaToAggregateDetails(codecRegistry: CodecRegistry, aggregate: AggregateMetadata): AggregateDetails = {
val sFunc: FunctionSummary = functionMetaToFunctionSummary(aggregate.getStateFunc)
val finalFunc: Option[String] = Option(aggregate.getFinalFunc).map(func => {
val finalFunction = functionMetaToFunctionSummary(func)
finalFunction.name + finalFunction.arguments.mkString("(", ", ", ")")
})
val sType = aggregate.getStateType
val initCond: Option[String] = Option(aggregate.getInitCond).map(codecRegistry.codecFor(sType).format(_))
val returnType: String = Option(aggregate.getFinalFunc) match {
case Some(finalFunc) => functionMetaToFunctionSummary(finalFunc).returnType
case None => sFunc.returnType
}
indexOption.map(index => IndexDetails(index.getName, buildIndexInfo(index)))
new AggregateDetails(aggregate.getKeyspace.getName,
aggregate.getSimpleName,
aggregate.getArgumentTypes.asScala.toList.map(_.asFunctionParameterString()),
sFunc.name + sFunc.arguments.mkString("(",", ", ")"),
sType.asFunctionParameterString(),
finalFunc,
initCond,
returnType,
aggregate.exportAsString())
}
private def extractStaticColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == true)
.map(c => new ColumnDetails(c.columnMeta.getName, StaticColumn, c.columnMeta.getType, extractIndexDetail(c)))
def aggregateMetaToAggregateSummary(aggregate: AggregateMetadata): AggregateSummary = {
val returnType: String = Option(aggregate.getFinalFunc) match {
case Some(finalFunc) => functionMetaToFunctionSummary(finalFunc).returnType
case None => aggregate.getStateType.asFunctionParameterString()
}
new AggregateSummary(aggregate.getKeyspace.getName,
aggregate.getSimpleName,
aggregate.getArgumentTypes.asScala.toList.map(_.asFunctionParameterString()),
returnType
)
}
private def convertClusteringColumns(columns: List[ColumnMetaWrapper], orders: List[TableMetadata.Order]): List[ColumnDetails] = {
columns
.zip(orders)
.map{case(c,order) => new ColumnDetails(c.columnMeta.getName,
new ClusteringColumn(OrderConverter.convert(order)),
c.columnMeta.getType, extractIndexDetail(c))}
def mvMetaToMaterializedViewDetails(mv: MaterializedViewMetadata): MaterializedViewDetails = {
new MaterializedViewDetails(mv.getName, MV.mvMetaToColumnDetails(mv), mv.exportAsString(), mv.getBaseTable.getName)
}
def mvMetaToMaterializedViewSummary(mv: MaterializedViewMetadata): MaterializedViewSummary = {
new MaterializedViewSummary(mv.getName, mv.getBaseTable.getName)
}
trait TableOrView {
protected def extractNormalColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == false)
.map(c => new ColumnDetails(c.columnMeta.getName, NormalColumn, c.columnMeta.getType))
}
protected def extractStaticColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == true)
.map(c => new ColumnDetails(c.columnMeta.getName, StaticColumn, c.columnMeta.getType))
}
protected def convertClusteringColumns(columns: List[ColumnMetaWrapper], orders: List[DriverClusteringOrder]): List[ColumnDetails] = {
columns
.zip(orders)
.map{case(c,order) => new ColumnDetails(c.columnMeta.getName,
new ClusteringColumn(OrderConverter.convert(order)),c.columnMeta.getType)}
}
protected def convertPartitionKeys(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.map(c => new ColumnDetails(c.columnMeta.getName, PartitionKey, c.columnMeta.getType))
}
}
object Table extends TableOrView {
def tableMetaToColumnDetails(meta: TableMetadata): List[ColumnDetails] = {
val partitionKeys: List[ColumnMetaWrapper] = meta.getPartitionKey.asScala.toList.map(new ColumnMetaWrapper(_))
val clusteringColumns: List[ColumnMetaWrapper] = meta.getClusteringColumns.asScala.toList.map(new ColumnMetaWrapper(_))
val columns: List[ColumnMetaWrapper] = meta.getColumns.asScala.toList.map(new ColumnMetaWrapper(_))
.diff(partitionKeys).diff(clusteringColumns)
val clusteringOrders = meta.getClusteringOrder.asScala.toList
convertPartitionKeys(partitionKeys):::
extractStaticColumns(columns):::
convertClusteringColumns(clusteringColumns, clusteringOrders):::
extractNormalColumns(columns)
}
def tableMetaToIndexDetails(meta: TableMetadata): List[IndexDetails] = {
meta.getIndexes.asScala.toList
.map(index => IndexDetails(index.getName, index.getTarget, index.asCQLQuery()))
.sortBy(index => index.name)
}
}
private def convertPartitionKeys(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.map(c => new ColumnDetails(c.columnMeta.getName, PartitionKey, c.columnMeta.getType, extractIndexDetail(c)))
object MV extends TableOrView {
def mvMetaToColumnDetails(meta: MaterializedViewMetadata): List[ColumnDetails] = {
val partitionKeys: List[ColumnMetaWrapper] = meta.getPartitionKey.asScala.toList.map(new ColumnMetaWrapper(_))
val clusteringColumns: List[ColumnMetaWrapper] = meta.getClusteringColumns.asScala.toList.map(new ColumnMetaWrapper(_))
val columns: List[ColumnMetaWrapper] = meta.getColumns.asScala.toList.map(new ColumnMetaWrapper(_))
.diff(partitionKeys).diff(clusteringColumns)
val clusteringOrders = meta.getClusteringOrder.asScala.toList
convertPartitionKeys(partitionKeys):::
convertClusteringColumns(clusteringColumns, clusteringOrders):::
extractNormalColumns(columns)
}
}
object UDT {
def userTypeToColumnDetails(userType: UserType): List[ColumnDetails] = {
userType.getFieldNames.asScala.toList
.map(name => new ColumnDetails(name, NormalColumn, userType.getFieldType(name)))
}
}
}

108
cassandra/src/main/scala/org/apache/zeppelin/cassandra/EnhancedSession.scala
View file

@ -20,6 +20,7 @@ import com.datastax.driver.core._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
import org.apache.zeppelin.interpreter.InterpreterException
import scala.collection.JavaConverters._
/**
* Enhance the Java driver session
@ -32,6 +33,9 @@ class EnhancedSession(val session: Session) {
val keyspaceDisplay = DisplaySystem.KeyspaceDisplay
val tableDisplay = DisplaySystem.TableDisplay
val udtDisplay = DisplaySystem.UDTDisplay
val functionDisplay = DisplaySystem.FunctionDisplay
val aggregateDisplay = DisplaySystem.AggregateDisplay
val materializedViewDisplay = DisplaySystem.MaterializedViewDisplay
val helpDisplay = DisplaySystem.HelpDisplay
private val noResultDisplay = DisplaySystem.NoResultDisplay
@ -61,8 +65,12 @@ class EnhancedSession(val session: Session) {
private def execute(describeKeyspace: DescribeKeyspaceCmd): String = {
val keyspace: String = describeKeyspace.keyspace
val metadata: KeyspaceMetadata = session.getCluster.getMetadata.getKeyspace(keyspace)
HTML_MAGIC + keyspaceDisplay.formatKeyspaceContent(describeKeyspace.statement, metadata)
val metadata: Option[KeyspaceMetadata] = Option(session.getCluster.getMetadata.getKeyspace(keyspace))
metadata match {
case Some(ksMeta) => HTML_MAGIC + keyspaceDisplay.formatKeyspaceContent(describeKeyspace.statement, ksMeta,
session.getCluster.getConfiguration.getCodecRegistry)
case None => throw new InterpreterException(s"Cannot find keyspace $keyspace")
}
}
private def execute(describeTable: DescribeTableCmd): String = {
@ -76,7 +84,7 @@ class EnhancedSession(val session: Session) {
}
}
private def execute(describeUDT: DescribeUDTCmd): String = {
private def execute(describeUDT: DescribeTypeCmd): String = {
val metaData = session.getCluster.getMetadata
val keyspace: String = describeUDT.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
val udtName: String = describeUDT.udtName
@ -87,6 +95,87 @@ class EnhancedSession(val session: Session) {
}
}
private def execute(describeUDTs: DescribeTypesCmd): String = {
val metadata: Metadata = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatAllUDTs(describeUDTs.statement, metadata)
}
private def execute(describeFunction: DescribeFunctionCmd): String = {
val metaData = session.getCluster.getMetadata
val keyspaceName: String = describeFunction.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
val functionName: String = describeFunction.function;
Option(metaData.getKeyspace(keyspaceName)) match {
case Some(keyspace) => {
val functionMetas: List[FunctionMetadata] = keyspace.getFunctions.asScala.toList
.filter(func => func.getSimpleName.toLowerCase == functionName.toLowerCase)
if(functionMetas.isEmpty) {
throw new InterpreterException(s"Cannot find function ${keyspaceName}.$functionName")
} else {
HTML_MAGIC + functionDisplay.format(describeFunction.statement, functionMetas, true)
}
}
case None => throw new InterpreterException(s"Cannot find function ${keyspaceName}.$functionName")
}
}
private def execute(describeFunctions: DescribeFunctionsCmd): String = {
val metadata: Metadata = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatAllFunctions(describeFunctions.statement, metadata)
}
private def execute(describeAggregate: DescribeAggregateCmd): String = {
val metaData = session.getCluster.getMetadata
val keyspaceName: String = describeAggregate.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
val aggregateName: String = describeAggregate.aggregate;
Option(metaData.getKeyspace(keyspaceName)) match {
case Some(keyspace) => {
val aggMetas: List[AggregateMetadata] = keyspace.getAggregates.asScala.toList
.filter(agg => agg.getSimpleName.toLowerCase == aggregateName.toLowerCase)
if(aggMetas.isEmpty) {
throw new InterpreterException(s"Cannot find aggregate ${keyspaceName}.$aggregateName")
} else {
HTML_MAGIC + aggregateDisplay.format(describeAggregate.statement, aggMetas, true,
session
.getCluster
.getConfiguration
.getCodecRegistry)
}
}
case None => throw new InterpreterException(s"Cannot find aggregate ${keyspaceName}.$aggregateName")
}
}
private def execute(describeAggregates: DescribeAggregatesCmd): String = {
val metadata: Metadata = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatAllAggregates(describeAggregates.statement, metadata)
}
private def execute(describeMV: DescribeMaterializedViewCmd): String = {
val metaData = session.getCluster.getMetadata
val keyspaceName: String = describeMV.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
val viewName: String = describeMV.view
Option(metaData.getKeyspace(keyspaceName)) match {
case Some(keyspace) => {
val viewMeta: Option[MaterializedViewMetadata] = Option(keyspace.getMaterializedView(viewName))
viewMeta match {
case Some(vMeta) => HTML_MAGIC + materializedViewDisplay.format(describeMV.statement, vMeta, true)
case None => throw new InterpreterException(s"Cannot find materialized view ${keyspaceName}.$viewName")
}
}
case None => throw new InterpreterException(s"Cannot find materialized view ${keyspaceName}.$viewName")
}
}
private def execute(describeMVs: DescribeMaterializedViewsCmd): String = {
val metadata: Metadata = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatAllMaterializedViews(describeMVs.statement, metadata)
}
private def execute(helpCmd: HelpCmd): String = {
HTML_MAGIC + helpDisplay.formatHelp()
}
@ -95,11 +184,18 @@ class EnhancedSession(val session: Session) {
def execute(st: Any): Any = {
st match {
case x:DescribeClusterCmd => execute(x)
case x:DescribeKeyspacesCmd => execute(x)
case x:DescribeTablesCmd => execute(x)
case x:DescribeKeyspaceCmd => execute(x)
case x:DescribeKeyspacesCmd => execute(x)
case x:DescribeTableCmd => execute(x)
case x:DescribeUDTCmd => execute(x)
case x:DescribeTablesCmd => execute(x)
case x:DescribeTypeCmd => execute(x)
case x:DescribeTypesCmd => execute(x)
case x:DescribeFunctionCmd => execute(x)
case x:DescribeFunctionsCmd => execute(x)
case x:DescribeAggregateCmd => execute(x)
case x:DescribeAggregatesCmd => execute(x)
case x:DescribeMaterializedViewCmd => execute(x)
case x:DescribeMaterializedViewsCmd => execute(x)
case x:HelpCmd => execute(x)
case x:Statement => session.execute(x)
case _ => throw new InterpreterException(s"Cannot execute statement '$st' of type ${st.getClass}")

25
cassandra/src/main/scala/org/apache/zeppelin/cassandra/InterpreterLogic.scala
View file

@ -100,7 +100,7 @@ class InterpreterLogic(val session: Session) {
logger.info(s"Executing CQL statements : \n\n$stringStatements\n")
try {
val protocolVersion = session.getCluster.getConfiguration.getProtocolOptions.getProtocolVersionEnum
val protocolVersion = session.getCluster.getConfiguration.getProtocolOptions.getProtocolVersion
val queries:List[AnyBlock] = parseInput(stringStatements)
@ -137,12 +137,12 @@ class InterpreterLogic(val session: Session) {
case x:BatchStm => {
val builtStatements: List[Statement] = x.statements.map {
case st:SimpleStm => generateSimpleStatement(st, queryOptions, context)
case st:BoundStm => generateBoundStatement(st, queryOptions, context)
case st:BoundStm => generateBoundStatement(session, st, queryOptions, context)
case _ => throw new InterpreterException(s"Unknown statement type")
}
generateBatchStatement(x.batchType, queryOptions, builtStatements)
}
case x:BoundStm => generateBoundStatement(x, queryOptions, context)
case x:BoundStm => generateBoundStatement(session, x, queryOptions, context)
case x:DescribeCommandStatement => x
case x:HelpCmd => x
case x => throw new InterpreterException(s"Unknown statement type : ${x}")
@ -208,7 +208,7 @@ class InterpreterLogic(val session: Session) {
row => {
val data = columnsDefinitions.map {
case (name, dataType) => {
if (row.isNull(name)) null else dataType.deserialize(row.getBytesUnsafe(name), protocolVersion)
if (row.isNull(name)) null else row.getObject(name)
}
}
output.append(data.mkString("\t")).append("\n")
@ -283,12 +283,12 @@ class InterpreterLogic(val session: Session) {
statement
}
def generateBoundStatement(st: BoundStm, options: CassandraQueryOptions,context: InterpreterContext): BoundStatement = {
def generateBoundStatement(session: Session, st: BoundStm, options: CassandraQueryOptions,context: InterpreterContext): BoundStatement = {
logger.debug(s"Generating bound statement with name : '${st.name}' and bound values : ${st.values}")
preparedStatements.get(st.name) match {
case Some(ps) => {
val boundValues = maybeExtractVariables(st.values, context)
createBoundStatement(st.name, ps, boundValues)
createBoundStatement(session.getCluster.getConfiguration.getCodecRegistry, st.name, ps, boundValues)
}
case None => throw new InterpreterException(s"The statement '${st.name}' can not be bound to values. " +
s"Are you sure you did prepare it with @prepare[${st.name}] ?")
@ -342,7 +342,7 @@ class InterpreterLogic(val session: Session) {
options.fetchSize.foreach(statement.setFetchSize(_))
}
private def createBoundStatement(name: String, ps: PreparedStatement, rawBoundValues: String): BoundStatement = {
private def createBoundStatement(codecRegistry: CodecRegistry, name: String, ps: PreparedStatement, rawBoundValues: String): BoundStatement = {
val dataTypes = ps.getVariables.toList
.map(cfDef => cfDef.getType)
@ -357,6 +357,7 @@ class InterpreterLogic(val session: Session) {
if(value.trim == "null") {
null
} else {
val codec: TypeCodec[AnyRef] = codecRegistry.codecFor[AnyRef](dataType)
dataType.getName match {
case (ASCII | TEXT | VARCHAR) => value.trim.replaceAll("(?<!')'","")
case (INT | VARINT) => value.trim.toInt
@ -369,11 +370,11 @@ class InterpreterLogic(val session: Session) {
case INET => InetAddress.getByName(value.trim)
case TIMESTAMP => parseDate(value.trim)
case (UUID | TIMEUUID) => java.util.UUID.fromString(value.trim)
case LIST => dataType.parse(boundValuesParser.parse(boundValuesParser.list, value).get)
case SET => dataType.parse(boundValuesParser.parse(boundValuesParser.set, value).get)
case MAP => dataType.parse(boundValuesParser.parse(boundValuesParser.map, value).get)
case UDT => dataType.parse(boundValuesParser.parse(boundValuesParser.udt, value).get)
case TUPLE => dataType.parse(boundValuesParser.parse(boundValuesParser.tuple, value).get)
case LIST => codec.parse(boundValuesParser.parse(boundValuesParser.list, value).get)
case SET => codec.parse(boundValuesParser.parse(boundValuesParser.set, value).get)
case MAP => codec.parse(boundValuesParser.parse(boundValuesParser.map, value).get)
case UDT => codec.parse(boundValuesParser.parse(boundValuesParser.udt, value).get)
case TUPLE => codec.parse(boundValuesParser.parse(boundValuesParser.tuple, value).get)
case _ => throw new InterpreterException(s"Cannot parse data of type : ${dataType.toString}")
}
}

31
cassandra/src/main/scala/org/apache/zeppelin/cassandra/MetaDataHierarchy.scala
View file

@ -28,10 +28,10 @@ import scala.util.parsing.json.JSONObject
*/
object MetaDataHierarchy {
object OrderConverter {
def convert(clusteringOrder: TableMetadata.Order): ClusteringOrder = {
def convert(clusteringOrder: com.datastax.driver.core.ClusteringOrder): ClusteringOrder = {
clusteringOrder match {
case TableMetadata.Order.ASC => ASC
case TableMetadata.Order.DESC => DESC
case com.datastax.driver.core.ClusteringOrder.ASC => ASC
case com.datastax.driver.core.ClusteringOrder.DESC => DESC
}
}
}
@ -46,9 +46,8 @@ object MetaDataHierarchy {
case class ClusteringColumn(order: ClusteringOrder) extends ColumnType
object StaticColumn extends ColumnType
object NormalColumn extends ColumnType
case class IndexDetails(name: String, info: String)
case class ColumnDetails(name: String, columnType: ColumnType, dataType: DataType, index: Option[IndexDetails])
case class IndexDetails(name: String, target: String, asCQL: String)
case class ColumnDetails(name: String, columnType: ColumnType, dataType: DataType)
case class ClusterDetails(name: String, partitioner: String)
case class ClusterContent(clusterName: String, clusterDetails: String, keyspaces: List[(UUID, String, String)])
@ -58,10 +57,26 @@ object MetaDataHierarchy {
JSONObject(replication).toString().replaceAll(""""""","'")
}
}
case class KeyspaceContent(keyspaceName: String, keyspaceDetails: String, tables: List[(UUID,String, String)], udts: List[(UUID, String, String)])
case class TableDetails(tableName: String, columns: List[ColumnDetails], asCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class KeyspaceContent(keyspaceName: String, keyspaceDetails: String,
tables: List[(UUID,String, String)],
views: List[(UUID,String, String)],
udts: List[(UUID, String, String)],
functions: List[(UUID, String, String)],
aggregates: List[(UUID, String, String)])
case class TableDetails(tableName: String, columns: List[ColumnDetails], indices: List[IndexDetails], asCQL: String, indicesAsCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class UDTDetails(typeName: String, columns: List[ColumnDetails], asCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class SameNameFunctionDetails(functions: List[FunctionDetails])
case class FunctionDetails(keyspace:String, name: String, arguments: List[String], calledOnNullInput: Boolean, returnType: String,
language:String, body: String, asCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class FunctionSummary(keyspace:String, name: String, arguments: List[String], returnType: String)
case class AggregateDetails(keyspace:String, name: String, arguments: List[String], sFunc: String, sType: String,
finalFunc: Option[String], initCond: Option[String], returnType: String,
asCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class AggregateSummary(keyspace:String, name: String, arguments: List[String], returnType: String)
case class SameNameAggregateDetails(aggregates: List[AggregateDetails])
case class MaterializedViewDetails(name: String, columns: List[ColumnDetails], asCQL: String, baseTable: String, uniqueId: UUID = UUIDs.timeBased())
case class MaterializedViewSummary(name: String, baseTable: String)
}

200
cassandra/src/main/scala/org/apache/zeppelin/cassandra/ParagraphParser.scala
View file

@ -19,10 +19,20 @@ package org.apache.zeppelin.cassandra
import com.datastax.driver.core._
import org.apache.zeppelin.cassandra.CassandraInterpreter._
import org.apache.zeppelin.interpreter.InterpreterException
import scala.util.matching.Regex
import scala.util.parsing.combinator._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
/**
* Parser using Scala combinator parsing
*
* (?i) means case-insensitive mode
* (?s) means DOT ALL mode
* (?is) means case-insensitive and DOT ALL mode
*
*/
object ParagraphParser {
val CONSISTENCY_LEVEL_PATTERN = ConsistencyLevel.values().toList
.map(_.name()).filter(!_.contains("SERIAL")).mkString("""^\s*@consistency\s*=\s*(""", "|" , """)\s*$""").r
@ -42,22 +52,43 @@ object ParagraphParser {
val BIND_PATTERN = """^\s*@bind\[([^]]+)\](?:=([^;]+))?""".r
val BATCH_PATTERN = """^(?i)\s*BEGIN\s+(UNLOGGED|COUNTER)?\s*BATCH""".r
/**
* Very complicated RegExp
* (?: OR REPLACE)? -> optional presence of OR REPLACE
* .+? -> match ANY character in RELUCTANT mode
* (?:\s*|\n|\r|\f) -> white space OR line returns (\n, \r, \f)
* (?:\s*|\n|\r|\f)AS(?:\s*|\n|\r|\f) -> AS preceded and followed by white space or line return
* (?:'|\$\$) -> simple quote (') OR double dollar ($$) as source code separator
* (?:'|\$\$).+?(?:'|\$\$)\s*; ->
* source code separator (?:'|\$\$)
* followed by ANY character in RELUCTANT mode (.+?)
* followed by source code separator (?:'|\$\$)
* followed by optional white-space(s) (\s*)
* followed by semi-colon (;)
*/
val UDF_PATTERN = """(?is)\s*(CREATE(?:\s+OR REPLACE)?\s+FUNCTION(?:\s+IF\s+NOT\s+EXISTS)?.+?(?:\s+|\n|\r|\f)AS(?:\s+|\n|\r|\f)(?:'|\$\$).+?(?:'|\$\$)\s*;)""".r
val GENERIC_STATEMENT_PREFIX =
"""(?is)\s*(?:INSERT|UPDATE|DELETE|SELECT|CREATE|UPDATE|
|DROP|GRANT|REVOKE|TRUNCATE|LIST|USE)\s+""".r
val VALID_IDENTIFIER = "[a-z][a-z0-9_]*"
val DESCRIBE_CLUSTER_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+CLUSTER;\s*$""".r
val DESCRIBE_KEYSPACES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACES;\s*$""".r
val DESCRIBE_TABLES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+TABLES;\s*$""".r
val DESCRIBE_CLUSTER_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+CLUSTER\s*;\s*$""".r
val DESCRIBE_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_KEYSPACES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACES\s*;\s*$""".r
val DESCRIBE_TABLE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_TABLES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+TABLES\s*;\s*$""".r
val DESCRIBE_TYPE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s*(""" +
@ -65,6 +96,35 @@ object ParagraphParser {
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_TYPES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+TYPES\s*;\s*$""".r
val DESCRIBE_FUNCTION_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+FUNCTION\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_FUNCTION_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+FUNCTION\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_FUNCTIONS_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+FUNCTIONS\s*;\s*$""").r
val DESCRIBE_AGGREGATE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+AGGREGATE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_AGGREGATE_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+AGGREGATE\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_AGGREGATES_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+AGGREGATES\s*;\s*$""").r
val DESCRIBE_MATERIALIZED_VIEW_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+MATERIALIZED\s+VIEW\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_MATERIALIZED_VIEW_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+MATERIALIZED\s+VIEW\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_MATERIALIZED_VIEWS_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+MATERIALIZED\s+VIEWS\s*;\s*$""").r
val HELP_PATTERN = """^(?i)\s*HELP;\s*$""".r
}
@ -74,7 +134,10 @@ class ParagraphParser extends RegexParsers{
import ParagraphParser._
def singleLineComment: Parser[Comment] = """\s*#.*""".r ^^ {case text => Comment(text.trim.replaceAll("#",""))}
def singleLineCommentHash: Parser[Comment] = """\s*#.*""".r ^^ {case text => Comment(text.trim.replaceAll("#",""))}
def singleLineCommentDoubleSlashes: Parser[Comment] = """\s*//.*""".r ^^ {case text => Comment(text.trim.replaceAll("//",""))}
def singleLineComment: Parser[Comment] = singleLineCommentHash | singleLineCommentDoubleSlashes
def multiLineComment: Parser[Comment] = """(?s)/\*(.*)\*/""".r ^^ {case text => Comment(text.trim.replaceAll("""/\*""","").replaceAll("""\*/""",""))}
//Query parameters
@ -85,7 +148,10 @@ class ParagraphParser extends RegexParsers{
def fetchSize: Parser[FetchSize] = """\s*@fetchSize.+""".r ^^ {case x => extractFetchSize(x.trim)}
//Statements
def createFunctionStatement: Parser[SimpleStm] = UDF_PATTERN ^^{case x => extractUdfStatement(x.trim)}
def genericStatement: Parser[SimpleStm] = s"""$GENERIC_STATEMENT_PREFIX[^;]+;""".r ^^ {case x => extractSimpleStatement(x.trim)}
// def allStatement: Parser[SimpleStm] = udfStatement | genericStatement
def prepare: Parser[PrepareStm] = """\s*@prepare.+""".r ^^ {case x => extractPreparedStatement(x.trim)}
def removePrepare: Parser[RemovePrepareStm] = """\s*@remove_prepare.+""".r ^^ {case x => extractRemovePreparedStatement(x.trim)}
def bind: Parser[BoundStm] = """\s*@bind.+""".r ^^ {case x => extractBoundStatement(x.trim)}
@ -95,9 +161,17 @@ class ParagraphParser extends RegexParsers{
private def describeCluster: Parser[DescribeClusterCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+CLUSTER.*""".r ^^ {extractDescribeClusterCmd(_)}
private def describeKeyspaces: Parser[DescribeKeyspacesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACES.*""".r ^^ {extractDescribeKeyspacesCmd(_)}
private def describeTables: Parser[DescribeTablesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TABLES.*""".r ^^ {extractDescribeTablesCmd(_)}
private def describeTypes: Parser[DescribeTypesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TYPES.*""".r ^^ {extractDescribeTypesCmd(_)}
private def describeFunctions: Parser[DescribeFunctionsCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+FUNCTIONS.*""".r ^^ {extractDescribeFunctionsCmd(_)}
private def describeAggregates: Parser[DescribeAggregatesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+AGGREGATES.*""".r ^^ {extractDescribeAggregatesCmd(_)}
private def describeMaterializedViews: Parser[DescribeMaterializedViewsCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+MATERIALIZED\s+VIEWS.*""".r ^^ {extractDescribeMaterializedViewsCmd(_)}
private def describeKeyspace: Parser[DescribeKeyspaceCmd] = """\s*(?i)(?:DESCRIBE|DESC)\s+KEYSPACE\s+.+""".r ^^ {extractDescribeKeyspaceCmd(_)}
private def describeTable: Parser[DescribeTableCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s+.+""".r ^^ {extractDescribeTableCmd(_)}
private def describeType: Parser[DescribeUDTCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s+.*""".r ^^ {extractDescribeTypeCmd(_)}
private def describeType: Parser[DescribeTypeCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s+.*""".r ^^ {extractDescribeTypeCmd(_)}
private def describeFunction: Parser[DescribeFunctionCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+FUNCTION\s+.*""".r ^^ {extractDescribeFunctionCmd(_)}
private def describeAggregate: Parser[DescribeAggregateCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+AGGREGATE\s+.*""".r ^^ {extractDescribeAggregateCmd(_)}
private def describeMaterializedView: Parser[DescribeMaterializedViewCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+MATERIALIZED\s+VIEW\s+.*""".r ^^ {extractDescribeMaterializedViewCmd(_)}
//Help
private def helpCommand: Parser[HelpCmd] = """(?i)\s*HELP.*""".r ^^{extractHelpCmd(_)}
@ -114,8 +188,14 @@ class ParagraphParser extends RegexParsers{
case begin ~ cqls ~ end => BatchStm(extractBatchType(begin),cqls)}
def queries:Parser[List[AnyBlock]] = rep(singleLineComment | multiLineComment | consistency | serialConsistency |
timestamp | retryPolicy | fetchSize | removePrepare | prepare | bind | batch | describeCluster | describeKeyspaces |
describeTables | describeKeyspace | describeTable | describeType | helpCommand | genericStatement)
timestamp | retryPolicy | fetchSize | removePrepare | prepare | bind | batch | describeCluster |
describeKeyspace | describeKeyspaces |
describeTable | describeTables |
describeType | describeTypes |
describeFunction | describeFunctions |
describeAggregate | describeAggregates |
describeMaterializedView | describeMaterializedViews |
helpCommand | createFunctionStatement | genericStatement)
def extractConsistency(text: String): Consistency = {
text match {
@ -171,6 +251,13 @@ class ParagraphParser extends RegexParsers{
}
}
def extractUdfStatement(text: String): SimpleStm = {
text match {
case UDF_PATTERN(statement) => SimpleStm(statement)
case _ => throw new InterpreterException(s"Invalid statement '$text' for UDF creation. Did you forget to add ; (semi-colon) at the end of each CQL statement ?")
}
}
def extractPreparedStatement(text: String): PrepareStm = {
text match {
case PREPARE_STATEMENT_PATTERN(name,queryString) => PrepareStm(name.trim,queryString.trim)
@ -214,6 +301,14 @@ class ParagraphParser extends RegexParsers{
}
}
def extractDescribeKeyspaceCmd(text: String): DescribeKeyspaceCmd = {
text match {
case DESCRIBE_KEYSPACE_PATTERN(keyspace) => new DescribeKeyspaceCmd(keyspace)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE KEYSPACE. " +
s"""It should comply to the pattern: ${DESCRIBE_KEYSPACE_PATTERN.toString}""")
}
}
def extractDescribeKeyspacesCmd(text: String): DescribeKeyspacesCmd = {
text match {
case DESCRIBE_KEYSPACES_PATTERN() => new DescribeKeyspacesCmd
@ -222,6 +317,15 @@ class ParagraphParser extends RegexParsers{
}
}
def extractDescribeTableCmd(text: String): DescribeTableCmd = {
text match {
case DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeTableCmd(Option(keyspace),table)
case DESCRIBE_TABLE_PATTERN(table) => new DescribeTableCmd(Option.empty,table)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TABLE. " +
s"""It should comply to the patterns: ${DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TABLE_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeTablesCmd(text: String): DescribeTablesCmd = {
text match {
case DESCRIBE_TABLES_PATTERN() => new DescribeTablesCmd
@ -230,32 +334,74 @@ class ParagraphParser extends RegexParsers{
}
}
def extractDescribeKeyspaceCmd(text: String): DescribeKeyspaceCmd = {
def extractDescribeTypeCmd(text: String): DescribeTypeCmd = {
text match {
case DESCRIBE_KEYSPACE_PATTERN(keyspace) => new DescribeKeyspaceCmd(keyspace)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE KEYSPACE. " +
s"""It should comply to the pattern: ${DESCRIBE_KEYSPACE_PATTERN.toString}""")
}
}
def extractDescribeTableCmd(text: String): DescribeTableCmd = {
text match {
case DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeTableCmd(Option(keyspace),table)
case DESCRIBE_TABLE_PATTERN(table) => new DescribeTableCmd(Option.empty,table)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TABLE. " +
s"""It should comply to the patterns: ${DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TABLE_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeTypeCmd(text: String): DescribeUDTCmd = {
text match {
case DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeUDTCmd(Option(keyspace),table)
case DESCRIBE_TYPE_PATTERN(table) => new DescribeUDTCmd(Option.empty,table)
case DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeTypeCmd(Option(keyspace),table)
case DESCRIBE_TYPE_PATTERN(table) => new DescribeTypeCmd(Option.empty,table)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TYPE. " +
s"""It should comply to the patterns: ${DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TYPE_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeTypesCmd(text: String): DescribeTypesCmd = {
text match {
case DESCRIBE_TYPES_PATTERN() => new DescribeTypesCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TYPES. " +
s"""It should comply to the pattern: ${DESCRIBE_TYPES_PATTERN.toString}""")
}
}
def extractDescribeFunctionCmd(text: String): DescribeFunctionCmd = {
text match {
case DESCRIBE_FUNCTION_WITH_KEYSPACE_PATTERN(keyspace,function) => new DescribeFunctionCmd(Option(keyspace),function)
case DESCRIBE_FUNCTION_PATTERN(function) => new DescribeFunctionCmd(Option.empty,function)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE FUNCTION. " +
s"""It should comply to the patterns: ${DESCRIBE_FUNCTION_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_FUNCTION_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeFunctionsCmd(text: String): DescribeFunctionsCmd = {
text match {
case DESCRIBE_FUNCTIONS_PATTERN() => new DescribeFunctionsCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE FUNCTIONS. " +
s"""It should comply to the pattern: ${DESCRIBE_FUNCTIONS_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeAggregateCmd(text: String): DescribeAggregateCmd = {
text match {
case DESCRIBE_AGGREGATE_WITH_KEYSPACE_PATTERN(keyspace,aggregate) => new DescribeAggregateCmd(Option(keyspace),aggregate)
case DESCRIBE_AGGREGATE_PATTERN(aggregate) => new DescribeAggregateCmd(Option.empty,aggregate)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE AGGREGATE. " +
s"""It should comply to the patterns: ${DESCRIBE_AGGREGATE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_AGGREGATE_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeAggregatesCmd(text: String): DescribeAggregatesCmd = {
text match {
case DESCRIBE_AGGREGATES_PATTERN() => new DescribeAggregatesCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE AGGREGATES. " +
s"""It should comply to the pattern: ${DESCRIBE_AGGREGATES_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeMaterializedViewCmd(text: String): DescribeMaterializedViewCmd = {
text match {
case DESCRIBE_MATERIALIZED_VIEW_WITH_KEYSPACE_PATTERN(keyspace,view) => new DescribeMaterializedViewCmd(Option(keyspace),view)
case DESCRIBE_MATERIALIZED_VIEW_PATTERN(view) => new DescribeMaterializedViewCmd(Option.empty,view)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE MATERIALIZED VIEW. " +
s"""It should comply to the patterns: ${DESCRIBE_MATERIALIZED_VIEW_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_MATERIALIZED_VIEW_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeMaterializedViewsCmd(text: String): DescribeMaterializedViewsCmd = {
text match {
case DESCRIBE_MATERIALIZED_VIEWS_PATTERN() => new DescribeMaterializedViewsCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE MATERIALIZED VIEWS. " +
s"""It should comply to the pattern: ${DESCRIBE_MATERIALIZED_VIEWS_PATTERN.toString}""".stripMargin)
}
}
def extractHelpCmd(text: String): HelpCmd = {
text match {
case HELP_PATTERN() => new HelpCmd

54
cassandra/src/main/scala/org/apache/zeppelin/cassandra/TextBlockHierarchy.scala
View file

@ -77,10 +77,16 @@ object TextBlockHierarchy {
object BatchStatementType extends StatementType
object DescribeClusterStatementType extends StatementType
object DescribeAllKeyspacesStatementType extends StatementType
object DescribeKeyspaceStatementType extends StatementType
object DescribeAllTablesStatementType extends StatementType
object DescribeAllTypesStatementType extends StatementType
object DescribeAllFunctionsStatementType extends StatementType
object DescribeAllAggregatesStatementType extends StatementType
object DescribeKeyspaceStatementType extends StatementType
object DescribeTableStatementType extends StatementType
object DescribeTypeStatementType extends StatementType
object DescribeFunctionStatementType extends StatementType
object DescribeAggregateStatementType extends StatementType
object DescribeMaterializedView extends StatementType
object HelpStatementType extends StatementType
abstract class QueryStatement(val statementType: StatementType) extends AnyBlock(StatementBlock) {
@ -104,15 +110,27 @@ object TextBlockHierarchy {
val statement: String
}
class DescribeClusterCmd(override val statement: String = "DESCRIBE CLUSTER;")
case class DescribeClusterCmd(override val statement: String = "DESCRIBE CLUSTER;")
extends QueryStatement(DescribeClusterStatementType) with DescribeCommandStatement
class DescribeKeyspacesCmd(override val statement: String = "DESCRIBE KEYSPACES;")
case class DescribeKeyspacesCmd(override val statement: String = "DESCRIBE KEYSPACES;")
extends QueryStatement(DescribeAllKeyspacesStatementType) with DescribeCommandStatement
class DescribeTablesCmd(override val statement: String = "DESCRIBE TABLES;")
case class DescribeTablesCmd(override val statement: String = "DESCRIBE TABLES;")
extends QueryStatement(DescribeAllTablesStatementType) with DescribeCommandStatement
case class DescribeTypesCmd(override val statement: String = "DESCRIBE TYPES;")
extends QueryStatement(DescribeAllTypesStatementType) with DescribeCommandStatement
case class DescribeFunctionsCmd(override val statement: String = "DESCRIBE FUNCTIONS;") extends QueryStatement(DescribeAllFunctionsStatementType)
with DescribeCommandStatement
case class DescribeAggregatesCmd(override val statement: String = "DESCRIBE AGGREGATES;") extends QueryStatement(DescribeAllAggregatesStatementType)
with DescribeCommandStatement
case class DescribeMaterializedViewsCmd(override val statement: String = "DESCRIBE MATERIALIZED VIEWS;") extends QueryStatement(DescribeAllAggregatesStatementType)
with DescribeCommandStatement
case class DescribeKeyspaceCmd(keyspace: String) extends QueryStatement(DescribeKeyspaceStatementType)
with DescribeCommandStatement {
override val statement: String = s"DESCRIBE KEYSPACE $keyspace;"
@ -126,7 +144,7 @@ object TextBlockHierarchy {
}
}
case class DescribeUDTCmd(keyspace:Option[String],udtName: String) extends QueryStatement(DescribeTypeStatementType)
case class DescribeTypeCmd(keyspace:Option[String], udtName: String) extends QueryStatement(DescribeTypeStatementType)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE TYPE $ks.$udtName;"
@ -134,6 +152,30 @@ object TextBlockHierarchy {
}
}
class HelpCmd extends QueryStatement(HelpStatementType)
case class DescribeFunctionCmd(keyspace:Option[String], function: String) extends QueryStatement(DescribeFunctionStatementType)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE FUNCTION $ks.$function;"
case None => s"DESCRIBE FUNCTION $function;"
}
}
case class DescribeAggregateCmd(keyspace:Option[String], aggregate: String) extends QueryStatement(DescribeAggregateStatementType)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE AGGREGATE $ks.$aggregate;"
case None => s"DESCRIBE AGGREGATE $aggregate;"
}
}
case class DescribeMaterializedViewCmd(keyspace:Option[String], view: String) extends QueryStatement(DescribeMaterializedView)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE MATERIALIZED VIEW $ks.$view;"
case None => s"DESCRIBE MATERIALIZED VIEW $view;"
}
}
case class HelpCmd(val statement:String = "HELP;") extends QueryStatement(HelpStatementType)
}

114
cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java
View file

@ -30,22 +30,20 @@ import static org.assertj.core.api.Assertions.*;
import static org.mockito.Mockito.when;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.ProtocolVersion;
import com.datastax.driver.core.Session;
import info.archinnov.achilles.junit.AchillesResource;
import info.archinnov.achilles.junit.AchillesResourceBuilder;
import info.archinnov.achilles.embedded.CassandraEmbeddedServerBuilder;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.*;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import scala.io.Source;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
@ -58,16 +56,14 @@ public class CassandraInterpreterTest {
private static final String ARTISTS_TABLE = "zeppelin.artists";
@ClassRule
public static AchillesResource resource = AchillesResourceBuilder
public static Session session = CassandraEmbeddedServerBuilder
.noEntityPackages()
.withKeyspaceName("zeppelin")
.withScript("prepare_schema.cql")
.withScript("prepare_data.cql")
.build();
private static Session session = resource.getNativeSession();
.withProtocolVersion(ProtocolVersion.V3)
.buildNativeSessionOnly();
// public static Session session = null;
private static CassandraInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
@ -76,7 +72,8 @@ public class CassandraInterpreterTest {
@BeforeClass
public static void setUp() {
Properties properties = new Properties();
final Cluster cluster = resource.getNativeSession().getCluster();
final Cluster cluster = session.getCluster();
// final Cluster cluster = null;
properties.setProperty(CASSANDRA_CLUSTER_NAME, cluster.getClusterName());
properties.setProperty(CASSANDRA_COMPRESSION_PROTOCOL, "NONE");
properties.setProperty(CASSANDRA_CREDENTIALS_USERNAME, "none");
@ -131,7 +128,7 @@ public class CassandraInterpreterTest {
@Test
public void should_create_cluster_and_session_upon_call_to_open() throws Exception {
assertThat(interpreter.cluster).isNotNull();
assertThat(interpreter.cluster.getClusterName()).isEqualTo(resource.getNativeSession().getCluster().getClusterName());
assertThat(interpreter.cluster.getClusterName()).isEqualTo(session.getCluster().getClusterName());
assertThat(interpreter.session).isNotNull();
assertThat(interpreter.helper).isNotNull();
}
@ -241,7 +238,7 @@ public class CassandraInterpreterTest {
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message())
.contains("Not enough replica available for query at consistency THREE (3 required but only 1 alive)");
.contains("Not enough replicas available for query at consistency THREE (3 required but only 1 alive)");
}
@Test
@ -329,14 +326,14 @@ public class CassandraInterpreterTest {
assertThat(actual.message()).isEqualTo(
"login\taddresses\tage\tdeceased\tfirstname\tlast_update\tlastname\tlocation\n" +
"jdoe\t" +
"{street_number:3, street_name:'Beverly Hills Bld', zip_code:90209," +
" country:'USA', extra_info:['Right on the hills', 'Next to the post box'], " +
"phone_numbers:{'office':2015790847, 'home':2016778524}}\tnull\t" +
"{street_number:3,street_name:'Beverly Hills Bld',zip_code:90209," +
"country:'USA',extra_info:['Right on the hills','Next to the post box']," +
"phone_numbers:{'office':2015790847,'home':2016778524}}\tnull\t" +
"null\t" +
"John\t" +
"null\t" +
"DOE\t" +
"('USA', 90209, 'Beverly Hills')\n");
"('USA',90209,'Beverly Hills')\n");
}
@Test
@ -553,6 +550,76 @@ public class CassandraInterpreterTest {
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
@Ignore
//TODO activate test when using Java 8 and C* 3.x
public void should_describe_function() throws Exception {
//Given
Properties properties = new Properties();
properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1");
properties.setProperty(CASSANDRA_PORT, "9042");
Interpreter interpreter = new CassandraInterpreter(properties);
interpreter.open();
String createFunction = "CREATE FUNCTION zeppelin.maxof(val1 int,val2 int) " +
"RETURNS NULL ON NULL INPUT " +
"RETURNS int " +
"LANGUAGE java " +
"AS $$" +
" return Math.max(val1, val2);\n" +
"$$;";
interpreter.interpret(createFunction, intrContext);
String query = "DESCRIBE FUNCTION zeppelin.maxOf;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("xxxxx");
}
@Test
@Ignore
//TODO activate test when using Java 8 and C* 3.x
public void should_describe_aggregate() throws Exception {
//Given
Properties properties = new Properties();
properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1");
properties.setProperty(CASSANDRA_PORT, "9042");
Interpreter interpreter = new CassandraInterpreter(properties);
interpreter.open();
final String query = "DESCRIBE AGGREGATES;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
@Test
@Ignore
//TODO activate test when using Java 8 and C* 3.x
public void should_describe_materialized_view() throws Exception {
//Given
Properties properties = new Properties();
properties.setProperty(CASSANDRA_HOSTS, "127.0.0.1");
properties.setProperty(CASSANDRA_PORT, "9042");
Interpreter interpreter = new CassandraInterpreter(properties);
interpreter.open();
final String query = "DESCRIBE MATERIALIZED VIEWS;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
@Test
public void should_describe_table() throws Exception {
//Given
@ -601,6 +668,7 @@ public class CassandraInterpreterTest {
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_error_describing_non_existing_table() throws Exception {
//Given
@ -647,8 +715,8 @@ public class CassandraInterpreterTest {
return rawHtml
.replaceAll("\\s*\n\\s*","")
.replaceAll(">\\s+<", "><")
.replaceAll("(?s)data-target=\"#[a-f0-9-]+(?:_asCQL)?\"", "")
.replaceAll("(?s)id=\"[a-f0-9-]+(?:_asCQL)?\"", "")
.replaceAll("(?s)data-target=\"#[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "")
.replaceAll("(?s)id=\"[a-f0-9-]+(?:_asCQL|_indices_asCQL)?\"", "")
.trim();
}

4
cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java
View file

@ -302,10 +302,10 @@ public class InterpreterLogicTest {
}
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
return scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toList();
return scala.collection.JavaConversions.asScalaIterable(list).toList();
}
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
return scala.collection.JavaConverters.seqAsJavaListConverter(list).asJava();
return scala.collection.JavaConversions.asJavaList(list);
}
}

99
cassandra/src/test/resources/scalate/DescribeCluster.html
View file

@ -1,98 +1 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE CLUSTER;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-4"></div>
<div class="col-md-4 col-offset-md-4">
<div class="table-responsive table-bordered">
<table class="table">
<caption>
<h4 class="text-muted">
<i class="glyphicon glyphicon-dashboard"/>&nbsp;Test Cluster
</h4>
</caption>
<thead>
<tr>
<th>Partitioner</th>
</tr>
</thead>
<tbody>
<tr>
<td>org.apache.cassandra.dht.Murmur3Partitioner</td>
</tr>
<tbody>
</table>
</div>
</div>
</div>
<br/><br/><nav class="navbar navbar-default"><ul class="nav navbar-nav"><li><a><strong>DESCRIBE CLUSTER;</strong></a></li></ul><ul class="nav navbar-nav navbar-right"><li class="dropdown"><a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><strong>Legend</strong><span class="caret"></span></a><ul class="dropdown-menu"><li><a role="button"><i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster</a></li><li><a role="button"><i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace</a></li><li><a role="button"><i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT</a></li><li><a role="button"><i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table</a></li><li><a role="button"><i class="glyphicon glyphicon-eye-open text-primary" />&nbsp;&nbsp;Materialized View</a></li><li><a role="button"><i class="glyphicon glyphicon-random text-success" />&nbsp;&nbsp;Function</a></li><li><a role="button"><i class="glyphicon glyphicon-retweet text-success" />&nbsp;&nbsp;Aggregate</a></li><li role="separator" class="divider text-muted"></li><li class="dropdown-header"><span class="text-primary">Table icons</span></li><li class="bg-info"><a role="button"><i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key</a></li><li class="bg-warning"><a role="button"><i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC</a></li></ul></li><li><a href="#"></a></li></ul></nav><hr/><div class="row"><div class="col-md-4"></div><div class="col-md-4 col-offset-md-4"><div class="table-responsive table-bordered"><table class="table"><caption><h4 class="text-muted"><i class="glyphicon glyphicon-dashboard"/>&nbsp;Test Cluster</h4></caption><thead><tr><th>Partitioner</th></tr></thead><tbody><tr><td>org.apache.cassandra.dht.Murmur3Partitioner</td></tr><tbody></table></div></div></div>

820
cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html
View file

File diff suppressed because one or more lines are too long

401
cassandra/src/test/resources/scalate/DescribeKeyspaces.html
View file

File diff suppressed because one or more lines are too long

327
cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html
View file

File diff suppressed because one or more lines are too long

138
cassandra/src/test/resources/scalate/DescribeType_live_data_address.html
View file

@ -1,137 +1 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE TYPE live_data.address;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption>
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">number</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">street</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">zip</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">city</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">country</td>
<td class="col-md-6">text</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.address (
number int,
street text,
zip int,
city text,
country text
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>
<br/><br/><nav class="navbar navbar-default"><ul class="nav navbar-nav"><li><a><strong>DESCRIBE TYPE live_data.address;</strong></a></li></ul><ul class="nav navbar-nav navbar-right"><li class="dropdown"><a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><strong>Legend</strong><span class="caret"></span></a><ul class="dropdown-menu"><li><a role="button"><i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster</a></li><li><a role="button"><i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace</a></li><li><a role="button"><i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT</a></li><li><a role="button"><i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table</a></li><li><a role="button"><i class="glyphicon glyphicon-eye-open text-primary" />&nbsp;&nbsp;Materialized View</a></li><li><a role="button"><i class="glyphicon glyphicon-random text-success" />&nbsp;&nbsp;Function</a></li><li><a role="button"><i class="glyphicon glyphicon-retweet text-success" />&nbsp;&nbsp;Aggregate</a></li><li role="separator" class="divider text-muted"></li><li class="dropdown-header"><span class="text-primary">Table icons</span></li><li class="bg-info"><a role="button"><i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key</a></li><li class="bg-warning"><a role="button"><i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC</a></li></ul></li><li><a href="#"></a></li></ul></nav><hr/><div class="row"><div class="col-md-3"></div><div class="col-md-6 col-offset-md-3"><div class="panel panel-default table-responsive table-bordered"><table class="table"><caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption><thead><tr><th class="col-md-6">Column Name</th><th class="col-md-6">Data Type</th></tr></thead><tbody><tr><td class="col-md-6">number</td><td class="col-md-6">int</td></tr><tr><td class="col-md-6">street</td><td class="col-md-6">text</td></tr><tr><td class="col-md-6">zip</td><td class="col-md-6">int</td></tr><tr><td class="col-md-6">city</td><td class="col-md-6">text</td></tr><tr><td class="col-md-6">country</td><td class="col-md-6">text</td></tr><tbody></table><div class="panel-footer"><a data-toggle="collapse" class="text-warning"><strong>As CQL statement</strong><span class="caret"></span></a><br/><br/><div class="collapse" ><pre class="well">CREATE TYPE live_data.address (number int,street text,zip int,city text,country text);</pre></div></div></div></div><div class="col-md-3"></div></div>

138
cassandra/src/test/resources/scalate/DescribeType_live_data_address_within_current_keyspace.html
View file

@ -1,137 +1 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE TYPE address;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption>
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">number</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">street</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">zip</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">city</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">country</td>
<td class="col-md-6">text</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.address (
number int,
street text,
zip int,
city text,
country text
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>
<br/><br/><nav class="navbar navbar-default"><ul class="nav navbar-nav"><li><a><strong>DESCRIBE TYPE address;</strong></a></li></ul><ul class="nav navbar-nav navbar-right"><li class="dropdown"><a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><strong>Legend</strong><span class="caret"></span></a><ul class="dropdown-menu"><li><a role="button"><i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster</a></li><li><a role="button"><i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace</a></li><li><a role="button"><i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT</a></li><li><a role="button"><i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table</a></li><li><a role="button"><i class="glyphicon glyphicon-eye-open text-primary" />&nbsp;&nbsp;Materialized View</a></li><li><a role="button"><i class="glyphicon glyphicon-random text-success" />&nbsp;&nbsp;Function</a></li><li><a role="button"><i class="glyphicon glyphicon-retweet text-success" />&nbsp;&nbsp;Aggregate</a></li><li role="separator" class="divider text-muted"></li><li class="dropdown-header"><span class="text-primary">Table icons</span></li><li class="bg-info"><a role="button"><i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key</a></li><li class="bg-warning"><a role="button"><i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC</a></li><li class="bg-success"><a role="button"><i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC</a></li></ul></li><li><a href="#"></a></li></ul></nav><hr/><div class="row"><div class="col-md-3"></div><div class="col-md-6 col-offset-md-3"><div class="panel panel-default table-responsive table-bordered"><table class="table"><caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption><thead><tr><th class="col-md-6">Column Name</th><th class="col-md-6">Data Type</th></tr></thead><tbody><tr><td class="col-md-6">number</td><td class="col-md-6">int</td></tr><tr><td class="col-md-6">street</td><td class="col-md-6">text</td></tr><tr><td class="col-md-6">zip</td><td class="col-md-6">int</td></tr><tr><td class="col-md-6">city</td><td class="col-md-6">text</td></tr><tr><td class="col-md-6">country</td><td class="col-md-6">text</td></tr><tbody></table><div class="panel-footer"><a data-toggle="collapse" class="text-warning"><strong>As CQL statement</strong><span class="caret"></span></a><br/><br/><div class="collapse" ><pre class="well">CREATE TYPE live_data.address (number int,street text,zip int,city text,country text);</pre></div></div></div></div><div class="col-md-3"></div></div>

871
cassandra/src/test/resources/scalate/Help.html
View file

File diff suppressed because one or more lines are too long

562
cassandra/src/test/scala/org/apache/zeppelin/cassandra/ParagraphParserTest.scala
View file

@ -23,6 +23,8 @@ import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
import org.apache.zeppelin.cassandra.ParagraphParser._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
import scala.Option
class ParagraphParserTest extends FlatSpec
with BeforeAndAfterEach
with Matchers
@ -57,34 +59,46 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parse(parser.queries,query)
parsed.get should be(List(
SimpleStm("SELECT * FROM albums LIMIT 10;"),
BatchStm(BatchStatement.Type.UNLOGGED,
List(
SimpleStm("INSERT INTO users(id) VALUES(10);"),
BoundStm("test","'a',12.34")
)
),
SimpleStm("SELECT * FROM users LIMIT 10;"),
BatchStm(BatchStatement.Type.LOGGED,
List(
SimpleStm("Insert INTO users(id) VALUES(11);"),
SimpleStm("INSERT INTO users(id) VALUES(12);")
)
),
BoundStm("toto","'a',12.34"),
DescribeTableCmd(Option("zeppelin"),"users"),
DescribeKeyspaceCmd("zeppelin")
))
parsed should matchPattern {
case parser.Success(List(
SimpleStm("SELECT * FROM albums LIMIT 10;"),
BatchStm(BatchStatement.Type.UNLOGGED,
List(
SimpleStm("INSERT INTO users(id) VALUES(10);"),
BoundStm("test","'a',12.34")
)
),
SimpleStm("SELECT * FROM users LIMIT 10;"),
BatchStm(BatchStatement.Type.LOGGED,
List(
SimpleStm("Insert INTO users(id) VALUES(11);"),
SimpleStm("INSERT INTO users(id) VALUES(12);")
)
),
BoundStm("toto","'a',12.34"),
DescribeTableCmd(Some("zeppelin"),"users"),
DescribeKeyspaceCmd("zeppelin")
),_) =>
}
}
"Parser" should "parse single-line comment" in {
"Parser" should "parse hash single-line comment" in {
val query :CharSequence="""#This is a comment""".stripMargin
val parsed = parser.parseAll[Comment](parser.singleLineComment, query)
parsed.get should be(Comment("This is a comment"))
parsed should matchPattern {
case parser.Success(Comment("This is a comment"), _) =>
}
}
"Parser" should "parse double slashes single-line comment" in {
val query :CharSequence="""//This is another comment""".stripMargin
val parsed = parser.parseAll[Comment](parser.singleLineComment, query)
parsed should matchPattern {
case parser.Success(Comment("This is another comment"), _) =>
}
}
"Parser" should "parse multi-line comment" in {
val query:String =
@ -96,13 +110,17 @@ class ParagraphParserTest extends FlatSpec
""".stripMargin
val parsed = parser.parseAll(parser.multiLineComment, query)
parsed.get should be(Comment("This is a comment\nline1\nline2\nline3\n"))
parsed should matchPattern {
case parser.Success(Comment("This is a comment\nline1\nline2\nline3\n"), _) =>
}
}
"Parser" should "parse consistency level" in {
val query:String =""" @consistency=ONE""".stripMargin
val parsed = parser.parseAll(parser.consistency, query)
parsed.get should be(Consistency(ConsistencyLevel.ONE))
parsed should matchPattern {
case parser.Success(Consistency(ConsistencyLevel.ONE), _) =>
}
}
"Parser" should "fails parsing unknown consistency level" in {
@ -116,7 +134,9 @@ class ParagraphParserTest extends FlatSpec
"Parser" should "parse serial consistency level" in {
val query:String =""" @serialConsistency=LOCAL_SERIAL""".stripMargin
val parsed = parser.parseAll(parser.serialConsistency, query)
parsed.get should be(SerialConsistency(ConsistencyLevel.LOCAL_SERIAL))
parsed should matchPattern {
case parser.Success(SerialConsistency(ConsistencyLevel.LOCAL_SERIAL), _) =>
}
}
"Parser" should "fails parsing unknown serial consistency level" in {
@ -130,7 +150,8 @@ class ParagraphParserTest extends FlatSpec
"Parser" should "parse timestamp" in {
val query:String =""" @timestamp=111""".stripMargin
val parsed = parser.parseAll(parser.timestamp, query)
parsed.get should be(Timestamp(111L))
parsed should matchPattern {
case parser.Success(Timestamp(111L), _) => }
}
"Parser" should "fails parsing invalid timestamp" in {
@ -144,7 +165,7 @@ class ParagraphParserTest extends FlatSpec
"Parser" should "parse retry policy" in {
val query:String ="@retryPolicy="+CassandraInterpreter.DOWNGRADING_CONSISTENCY_RETRY
val parsed = parser.parseAll(parser.retryPolicy, query)
parsed.get should be(DowngradingRetryPolicy)
parsed should matchPattern {case parser.Success(DowngradingRetryPolicy, _) => }
}
"Parser" should "fails parsing invalid retry policy" in {
@ -158,7 +179,7 @@ class ParagraphParserTest extends FlatSpec
"Parser" should "parse fetch size" in {
val query:String ="@fetchSize=100"
val parsed = parser.parseAll(parser.fetchSize, query)
parsed.get should be(FetchSize(100))
parsed should matchPattern { case parser.Success(FetchSize(100), _) =>}
}
"Parser" should "fails parsing invalid fetch size" in {
@ -177,7 +198,7 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.genericStatement, query)
//Then
parsed.get should be(SimpleStm("sElecT * FROM users LIMIT ? ;"))
parsed should matchPattern { case parser.Success(SimpleStm("sElecT * FROM users LIMIT ? ;"), _) =>}
}
"Parser" should "parse prepare" in {
@ -188,7 +209,7 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.prepare, query)
//Then
parsed.get should be(PrepareStm("select_users","SELECT * FROM users LIMIT ?"))
parsed should matchPattern { case parser.Success(PrepareStm("select_users","SELECT * FROM users LIMIT ?"), _) => }
}
"Parser" should "fails parsing invalid prepared statement" in {
@ -207,7 +228,7 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.removePrepare, query)
//Then
parsed.get should be(RemovePrepareStm("select_users"))
parsed should matchPattern { case parser.Success(RemovePrepareStm("select_users"), _) => }
}
"Parser" should "fails parsing invalid remove prepared statement" in {
@ -226,7 +247,7 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.bind, query)
//Then
parsed.get should be(BoundStm("select_users","10,'toto'"))
parsed should matchPattern { case parser.Success(BoundStm("select_users","10,'toto'"), _) => }
}
"Parser" should "fails parsing invalid bind statement" in {
@ -251,17 +272,17 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.batch, query)
//Then
parsed.get should be(
BatchStm(
BatchStatement.Type.LOGGED,
List[QueryStatement](
SimpleStm("Insert INTO users(id) VALUES(10);"),
BoundStm("select_users", "10,'toto'"),
SimpleStm("update users SET name ='John DOE' WHERE id=10;"),
SimpleStm("dElEtE users WHERE id=11;")
parsed should matchPattern {
case parser.Success(BatchStm(
BatchStatement.Type.LOGGED,
List(
SimpleStm("Insert INTO users(id) VALUES(10);"),
BoundStm("select_users", "10,'toto'"),
SimpleStm("update users SET name ='John DOE' WHERE id=10;"),
SimpleStm("dElEtE users WHERE id=11;")
)
)
)
), _) =>
}
}
"Parser" should "fails parsing invalid batch type" in {
@ -280,10 +301,12 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SerialConsistency(ConsistencyLevel.SERIAL),
SimpleStm("SELECT * FROM zeppelin.artists LIMIT 1;")
))
parsed should matchPattern {
case parser.Success(List(
SerialConsistency(ConsistencyLevel.SERIAL),
SimpleStm("SELECT * FROM zeppelin.artists LIMIT 1;")
), _) =>
}
}
"Parser" should "parse multi-line single statement" in {
@ -292,13 +315,15 @@ class ParagraphParserTest extends FlatSpec
" title text PRIMARY KEY,\n" +
" artist text,\n" +
" year int\n" +
");\n";
");\n"
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);")
))
parsed should matchPattern {
case parser.Success(List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);")
), _) =>
}
}
"Parser" should "parse multi-line statements" in {
@ -316,11 +341,12 @@ class ParagraphParserTest extends FlatSpec
"APPLY BATCH;\n"+
"@timestamp=10\n" +
"@retryPolicy=DOWNGRADING_CONSISTENCY\n" +
"SELECT * FROM zeppelin.albums;";
"SELECT * FROM zeppelin.albums;"
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
parsed should matchPattern {
case parser.Success(List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);"),
Consistency(ConsistencyLevel.THREE),
SerialConsistency(ConsistencyLevel.SERIAL),
@ -334,7 +360,8 @@ class ParagraphParserTest extends FlatSpec
Timestamp(10L),
DowngradingRetryPolicy,
SimpleStm("SELECT * FROM zeppelin.albums;")
))
), _) =>
}
}
"Parser" should "parse mixed single-line and multi-line statements" in {
@ -349,11 +376,12 @@ class ParagraphParserTest extends FlatSpec
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);\n"+
"APPLY BATCH;"+
"SELECT * FROM zeppelin.albums;";
"SELECT * FROM zeppelin.albums;"
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
parsed should matchPattern {
case parser.Success(List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);"),
BatchStm(BatchStatement.Type.LOGGED,
List(
@ -363,7 +391,8 @@ class ParagraphParserTest extends FlatSpec
)
),
SimpleStm("SELECT * FROM zeppelin.albums;")
))
), _) =>
}
}
"Parser" should "parse a block queries with comments" in {
@ -395,7 +424,8 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
parsed should matchPattern {
case parser.Success(List(
Comment("\n This example show how to force a\n timestamp on the query\n "),
Comment("Timestamp in the past"),
Timestamp(10L),
@ -409,8 +439,9 @@ class ParagraphParserTest extends FlatSpec
SimpleStm("INSERT INTO spark_demo.ts(key,value) VALUES(1,'val2');"),
Comment("Check the result"),
SimpleStm("SELECT * FROM spark_demo.ts WHERE key=1;")
)
)
), _
) =>
}
}
"Parser" should "remove prepared statement" in {
@ -426,13 +457,14 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(
parsed should matchPattern {
case parser.Success(List(
Comment("Removing an unknown statement should has no side effect"),
RemovePrepareStm("unknown_statement"),
RemovePrepareStm("select_artist_by_name"),
Comment("This should fail because the 'select_artist_by_name' has been removed"),
BoundStm("select_artist_by_name","'The Beatles'")
))
), _) => }
}
"Parser" should "parse only parameter" in {
@ -441,7 +473,9 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(FetchSize(1000)))
parsed should matchPattern {
case parser.Success(List(FetchSize(1000)), _) =>
}
}
@ -450,7 +484,9 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeClusterCmd]
parsed should matchPattern {
case parser.Success(List(DescribeClusterCmd("DESCRIBE CLUSTER;")), _) =>
}
}
"Parser" should "fail parsing describe cluster" in {
@ -462,46 +498,14 @@ class ParagraphParserTest extends FlatSpec
ex.getMessage should be(s"Invalid syntax for DESCRIBE CLUSTER. It should comply to the pattern: ${DESCRIBE_CLUSTER_PATTERN.toString}")
}
"Parser" should "parse describe keyspaces" in {
val queries ="Describe KeYsPaCeS;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeKeyspacesCmd]
}
"Parser" should "fail parsing describe keyspaces" in {
val queries ="Describe KeYsPaCeS"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE KEYSPACES. It should comply to the pattern: ${DESCRIBE_KEYSPACES_PATTERN.toString}")
}
"Parser" should "parse describe tables" in {
val queries ="Describe TaBlEs;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeTablesCmd]
}
"Parser" should "fail parsing describe tables" in {
val queries ="Describe TaBlEs"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TABLES. It should comply to the pattern: ${DESCRIBE_TABLES_PATTERN.toString}")
}
"Parser" should "parse describe keyspace" in {
val queries ="Describe KeYsPaCe toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeKeyspaceCmd("toto")))
parsed should matchPattern {
case parser.Success(List(DescribeKeyspaceCmd("toto")), _) =>
}
}
"Parser" should "fail parsing describe keyspace" in {
@ -513,6 +517,25 @@ class ParagraphParserTest extends FlatSpec
ex.getMessage should be(s"Invalid syntax for DESCRIBE KEYSPACE. It should comply to the pattern: ${DESCRIBE_KEYSPACE_PATTERN.toString}")
}
"Parser" should "parse describe keyspaces" in {
val queries ="Describe KeYsPaCeS;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeKeyspacesCmd("DESCRIBE KEYSPACES;")), _) =>
}
}
"Parser" should "fail parsing describe keyspaces" in {
val queries ="Describe KeYsPaCeS"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE KEYSPACES. It should comply to the pattern: ${DESCRIBE_KEYSPACES_PATTERN.toString}")
}
"Parser" should "parse describe table" in {
val queries ="Describe TaBlE toto;"
@ -539,12 +562,33 @@ class ParagraphParserTest extends FlatSpec
s"${DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TABLE_PATTERN.toString}")
}
"Parser" should "parse describe tables" in {
val queries ="Describe TaBlEs;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeTablesCmd("DESCRIBE TABLES;")), _) =>
}
}
"Parser" should "fail parsing describe tables" in {
val queries ="Describe TaBlEs"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TABLES. It should comply to the pattern: ${DESCRIBE_TABLES_PATTERN.toString}")
}
"Parser" should "parse describe type" in {
val queries ="Describe Type toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeUDTCmd(None,"toto")))
parsed should matchPattern {
case parser.Success(List(DescribeTypeCmd(None, "toto")), _) =>
}
}
"Parser" should "parse describe type with keyspace" in {
@ -552,7 +596,9 @@ class ParagraphParserTest extends FlatSpec
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeUDTCmd(Some("ks"),"toto")))
parsed should matchPattern {
case parser.Success(List(DescribeTypeCmd(Some("ks"), "toto")), _) =>
}
}
"Parser" should "fail parsing describe type" in {
@ -565,12 +611,186 @@ class ParagraphParserTest extends FlatSpec
s"${DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TYPE_PATTERN.toString}")
}
"Parser" should "parse describe types" in {
val queries ="Describe types;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeTypesCmd("DESCRIBE TYPES;")), _) =>
}
}
"Parser" should "fail parsing describe types" in {
val queries ="Describe types"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TYPES. It should comply to the pattern: ${DESCRIBE_TYPES_PATTERN.toString}")
}
"Parser" should "parse describe function" in {
val queries ="Describe function toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeFunctionCmd(None,"toto")),_) =>
}
}
"Parser" should "parse describe function with keyspace" in {
val queries ="Describe function ks.toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeFunctionCmd(Some("ks"), "toto")), _) =>
}
}
"Parser" should "fail parsing describe function" in {
val queries ="Describe function toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE FUNCTION. It should comply to the patterns: " +
s"${DESCRIBE_FUNCTION_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_FUNCTION_PATTERN.toString}")
}
"Parser" should "parse describe functions" in {
val queries ="Describe functions;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeFunctionsCmd("DESCRIBE FUNCTIONS;")), _) =>
}
}
"Parser" should "fail parsing describe functions" in {
val queries ="Describe functions toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE FUNCTIONS. It should comply to the pattern: " +
s"${DESCRIBE_FUNCTIONS_PATTERN.toString}")
}
"Parser" should "parse describe aggregate" in {
val queries ="Describe aggregate toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeAggregateCmd(None, "toto")), _) =>
}
}
"Parser" should "parse describe aggregate with keyspace" in {
val queries ="Describe aggregate ks.toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeAggregateCmd(Some("ks"),"toto")), _) =>
}
}
"Parser" should "fail parsing describe aggregate" in {
val queries ="Describe aggregate toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE AGGREGATE. It should comply to the patterns: " +
s"${DESCRIBE_AGGREGATE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_AGGREGATE_PATTERN.toString}")
}
"Parser" should "parse describe aggregates" in {
val queries ="Describe aggregates;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeAggregatesCmd("DESCRIBE AGGREGATES;")), _) =>
}
}
"Parser" should "fail parsing describe aggregates" in {
val queries ="Describe aggregates toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE AGGREGATES. It should comply to the pattern: " +
s"${DESCRIBE_AGGREGATES_PATTERN.toString}")
}
"Parser" should "parse describe materialized view" in {
val queries ="Describe materialized view toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeMaterializedViewCmd(None, "toto")), _) =>
}
}
"Parser" should "parse describe materialized view with keyspace" in {
val queries ="Describe materialized view ks.toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeMaterializedViewCmd(Some("ks"), "toto")), _) =>
}
}
"Parser" should "fail parsing describe materialized view" in {
val queries ="Describe materialized view toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE MATERIALIZED VIEW. It should comply to the patterns: " +
s"${DESCRIBE_MATERIALIZED_VIEW_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_MATERIALIZED_VIEW_PATTERN.toString}")
}
"Parser" should "parse describe materialized views" in {
val queries ="Describe materialized views;"
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(DescribeMaterializedViewsCmd("DESCRIBE MATERIALIZED VIEWS;")), _) =>
}
}
"Parser" should "fail parsing describe materialized views" in {
val queries ="Describe materialized views toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE MATERIALIZED VIEWS. It should comply to the pattern: " +
s"${DESCRIBE_MATERIALIZED_VIEWS_PATTERN.toString}")
}
"Parser" should "parse help" in {
val queries ="hElp;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [HelpCmd]
parsed should matchPattern {
case parser.Success(List(HelpCmd("HELP;")), _) =>
}
}
"Parser" should "fail parsing help" in {
@ -582,4 +802,146 @@ class ParagraphParserTest extends FlatSpec
ex.getMessage should be(s"Invalid syntax for HELP. It should comply to the patterns: " +
s"${HELP_PATTERN.toString}")
}
"Parser" should "parse CREATE FUNCTION" in {
val query = "CREATE FUNCTION keyspace.udf xxx AS 'return true;';"
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
"Parser" should "parse CREATE OR REPLACE FUNCTION" in {
val query = "CREATE or Replace FUNCTION keyspace.udf xxx AS 'return true;';"
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
"Parser" should "parse CREATE FUNCTION IF NOT Exists" in {
val query = "CREATE FUNCTION IF NOT EXISTS keyspace.udf xxx AS 'return true;';"
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
"Parser" should "parse CREATE FUNCTION multiline with simple quote" in {
val query =
"""CREATE FUNCTION IF NOT EXISTS keyspace.udf(input text) xxx
| CALLED ON NULL INPUT
| RETURN text
| LANGUAGE java
| AS '
| return input.toLowerCase("abc");
| ';""".stripMargin
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
"Parser" should "parse CREATE FUNCTION multiline with double dollar" in {
val query =
"""CREATE FUNCTION IF NOT EXISTS keyspace.udf(input text) xxx
| CALLED ON NULL INPUT
| RETURN text
| LANGUAGE java
| AS $$
| return input.toLowerCase("abc");
| $$;""".stripMargin
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
"Parser" should "parse CREATE FUNCTION multiline with SELECT" in {
val udf =
"""CREATE FUNCTION IF NOT EXISTS keyspace.udf(input text) xxx
| CALLED ON NULL INPUT
| RETURN text
| LANGUAGE java
| AS '
| return input.toLowerCase("abc");
| ';""".stripMargin
val select = "SELECT udf(val) from table WHERe id=1;"
val queries =
s"""$udf
|$select
""".stripMargin
val parsed = parser.parseAll(parser.queries, queries)
parsed should matchPattern {
case parser.Success(List(SimpleStm(udf), SimpleStm(select)), _) =>
}
}
"Parser" should "parse CREATE multiple FUNCTIONS" in {
val udf1 =
"""CREATE FUNCTION IF NOT EXISTS keyspace.udf(input text) xxx
| CALLED ON NULL INPUT
| RETURN text
| LANGUAGE java
| AS '
| return input.toLowerCase("abc");
| ';""".stripMargin
val select = "SELECT * FROM keyspace.table;"
val udf2 = """CREATE FUNCTION IF NOT EXISTS keyspace.maxOf(val1 int, val2 int)
| CALLED ON NULL INPUT
| RETURN text
| LANGUAGE java
| AS '
| return Math.max(val1,val2);
| ';""".stripMargin
val queries =
s"""$udf1
|
|$select
|
|$udf2
""".stripMargin
val parsed = parser.parseAll(parser.queries, queries)
parsed.get.size should be(3)
parsed should matchPattern {
case parser.Success(List(SimpleStm(udf1), SimpleStm(select), SimpleStm(udf2)), _) =>
}
}
"Parser" should "parse CREATE Materialized View" in {
val query =
"""CREATE MATERIALIZED VIEW xxx
| AS SELECT * FROM myTable
| WHERE partition IS NOT NULL
| PRIMARY KEY(col, partition);""".stripMargin
val parsed = parser.parseAll(parser.queries, query)
parsed should matchPattern {
case parser.Success(List(SimpleStm(query)), _) =>
}
}
}

2
conf/zeppelin-site.xml.template
View file

@ -105,7 +105,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

4
dev/merge_zeppelin_pr.py
View file

@ -39,7 +39,7 @@ except ImportError:
# Location of your Zeppelin git development area
ZEPPELIN_HOME = os.environ.get("ZEPPELIN_HOME", os.getcwd())
# Remote name which points to the Gihub site
# Remote name which points to the Github site
PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
# Remote name which points to Apache git
PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
@ -352,5 +352,5 @@ if JIRA_IMPORTED:
print "JIRA_USERNAME and JIRA_PASSWORD not set"
print "Exiting without trying to close the associated JIRA."
else:
print "Could not find jira-python library. Run 'sudo pip install jira-python' to install."
print "Could not find jira library. Run 'sudo pip install jira' to install."
print "Exiting without trying to close the associated JIRA."

2
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -51,6 +51,7 @@
<li><a href="{{BASE_PATH}}/interpreter/scalding.html">Scalding</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Shell</a></li>
<li><a href="{{BASE_PATH}}/interpreter/spark.html">Spark</a></li>
<li><a href="{{BASE_PATH}}/interpreter/tachyon.html">Tachyon</a></li>
<li><a href="{{BASE_PATH}}/pleasecontribute.html">Tajo</a></li>
</ul>
</li>
@ -76,6 +77,7 @@
<!-- li><span><b>REST API</b><span></li -->
<li><a href="{{BASE_PATH}}/rest-api/rest-interpreter.html">Interpreter API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-notebook.html">Notebook API</a></li>
<li><a href="{{BASE_PATH}}/rest-api/rest-configuration.html">Configuration API</a></li>
<li role="separator" class="divider"></li>
<!-- li><span><b>Development</b><span></li -->
<li><a href="{{BASE_PATH}}/development/writingzeppelininterpreter.html">Writing Zeppelin Interpreter</a></li>

BIN
docs/assets/themes/zeppelin/img/docs-img/tachyon-example.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 43 KiB

979
docs/interpreter/cassandra.md
View file

File diff suppressed because it is too large Load diff

105
docs/interpreter/elasticsearch.md
View file

@ -6,13 +6,10 @@ group: manual
---
{% include JB/setup %}
## Elasticsearch Interpreter for Apache Zeppelin
[Elasticsearch](https://www.elastic.co/products/elasticsearch) is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements.
<br />
## 1. Configuration
## Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
@ -45,24 +42,20 @@ group: manual
![Interpreter configuration](../assets/themes/zeppelin/img/docs-img/elasticsearch-config.png)
</center>
> **Note #1 :** You can add more properties to configure the Elasticsearch client.
> **Note #2 :** If you use Shield, you can add a property named `shield.user` with a value containing the name and the password ( format: `username:password` ). For more details about Shield configuration, consult the [Shield reference guide](https://www.elastic.co/guide/en/shield/current/_using_elasticsearch_java_clients_with_shield.html). Do not forget, to copy the shield client jar in the interpreter directory (`ZEPPELIN_HOME/interpreters/elasticsearch`).
<br />
## 2. Enabling the Elasticsearch Interpreter
## Enabling the Elasticsearch Interpreter
In a notebook, to enable the **Elasticsearch** interpreter, click the **Gear** icon and select **Elasticsearch**.
<br />
## 3. Using the Elasticsearch Interpreter
## Using the Elasticsearch Interpreter
In a paragraph, use `%elasticsearch` to select the Elasticsearch interpreter and then input all commands. To get the list of available commands, use `help`.
```bash
| %elasticsearch
| help
%elasticsearch
help
Elasticsearch interpreter:
General format: <command> /<indices>/<types>/<id> <option> <JSON>
- indices: list of indices separated by commas (depends on the command)
@ -84,19 +77,17 @@ Commands:
> **Tip :** Use ( Ctrl + . ) for autocompletion.
### Get
With the `get` command, you can find a document by id. The result is a JSON document.
```bash
| %elasticsearch
| get /index/type/id
%elasticsearch
get /index/type/id
```
Example:
![Elasticsearch - Get](../assets/themes/zeppelin/img/docs-img/elasticsearch-get.png)
### Search
With the `search` command, you can send a search query to Elasticsearch. There are two formats of query:
@ -106,53 +97,50 @@ With the `search` command, you can send a search query to Elasticsearch. There a
* This is a shortcut to a query like that: `{ "query": { "query_string": { "query": "__HERE YOUR QUERY__", "analyze_wildcard": true } } }`
* See [Elasticsearch query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) for more details about the content of such a query.
```bash
| %elasticsearch
| search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
%elasticsearch
search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
```
If you want to modify the size of the result set, you can add a line that is setting the size, before your search command.
```bash
| %elasticsearch
| size 50
| search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
%elasticsearch
size 50
search /index1,index2,.../type1,type2,... <JSON document containing the query or query_string elements>
```
> A search query can also contain [aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html). If there is at least one aggregation, the result of the first aggregation is shown, otherwise, you get the search hits.
Examples:
* With a JSON query:
```bash
| %elasticsearch
| search / { "query": { "match_all": { } } }
|
| %elasticsearch
| search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
|
| %elasticsearch
| search /logs { "aggs": {
| "content_length_stats": {
| "extended_stats": {
| "field": "content_length"
| }
| }
| } }
%elasticsearch
search / { "query": { "match_all": { } } }
%elasticsearch
search /logs { "query": { "query_string": { "query": "request.method:GET AND status:200" } } }
%elasticsearch
search /logs { "aggs": {
"content_length_stats": {
"extended_stats": {
"field": "content_length"
}
}
} }
```
* With query_string elements:
```bash
| %elasticsearch
| search /logs request.method:GET AND status:200
|
| %elasticsearch
| search /logs (404 AND (POST OR DELETE))
%elasticsearch
search /logs request.method:GET AND status:200
%elasticsearch
search /logs (404 AND (POST OR DELETE))
```
> **Important** : a document in Elasticsearch is a JSON document, so it is hierarchical, not flat as a row in a SQL table.
@ -199,13 +187,12 @@ Examples:
* With a query containing a multi-bucket aggregation:
![Elasticsearch - Search with aggregation (multi-bucket)](../assets/themes/zeppelin/img/docs-img/elasticsearch-agg-multi-bucket-pie.png)
### Count
With the `count` command, you can count documents available in some indices and types. You can also provide a query.
```bash
| %elasticsearch
| count /index1,index2,.../type1,type2,... <JSON document containing the query OR a query string>
%elasticsearch
count /index1,index2,.../type1,type2,... <JSON document containing the query OR a query string>
```
Examples:
@ -216,34 +203,30 @@ Examples:
* With a query:
![Elasticsearch - Count with query](../assets/themes/zeppelin/img/docs-img/elasticsearch-count-with-query.png)
### Index
With the `index` command, you can insert/update a document in Elasticsearch.
```bash
| %elasticsearch
| index /index/type/id <JSON document>
|
| %elasticsearch
| index /index/type <JSON document>
%elasticsearch
index /index/type/id <JSON document>
%elasticsearch
index /index/type <JSON document>
```
### Delete
With the `delete` command, you can delete a document.
```bash
| %elasticsearch
| delete /index/type/id
%elasticsearch
delete /index/type/id
```
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```bash
| %elasticsearch
| size ${limit=10}
| search /index/type { "query": { "match_all": { } } }
%elasticsearch
size ${limit=10}
search /index/type { "query": { "match_all": { } } }
```

4
docs/interpreter/flink.md
View file

@ -6,11 +6,9 @@ group: manual
---
{% include JB/setup %}
## Flink interpreter for Apache Zeppelin
[Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing. Flink’s core is a streaming dataflow engine that provides data distribution, communication, and fault tolerance for distributed computations over data streams. Flink also builds batch processing on top of the streaming engine, overlaying native iteration support, managed memory, and program optimization.
<br>
## How to start local Flink cluster, to test the interpreter
Zeppelin comes with pre-configured flink-local interpreter, which starts Flink in a local mode on your machine, so you do not need to install anything.
@ -38,10 +36,8 @@ At the "Interpreters" menu, you have to create a new Flink interpreter and provi
For more information about Flink configuration, you can find it [here](https://ci.apache.org/projects/flink/flink-docs-release-0.10/setup/config.html).
## How to test it's working
In example, by using the [Zeppelin notebook](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL05GTGFicy96ZXBwZWxpbi1ub3RlYm9va3MvbWFzdGVyL25vdGVib29rcy8yQVFFREs1UEMvbm90ZS5qc29u) is from Till Rohrmann's presentation [Interactive data analysis with Apache Flink](http://www.slideshare.net/tillrohrmann/data-analysis-49806564) for Apache Flink Meetup.
```
%sh
rm 10.txt.utf-8

60
docs/interpreter/geode.md
View file

@ -6,10 +6,7 @@ group: manual
---
{% include JB/setup %}
## Geode/Gemfire OQL Interpreter for Apache Zeppelin
<br/>
<table class="table-configuration">
<tr>
<th>Name</th>
@ -23,7 +20,6 @@ group: manual
</tr>
</table>
<br/>
This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Object Query Language (OQL)](http://geode-docs.cfapps.io/docs/developing/querying_basics/oql_compared_to_sql.html). With the OQL-based querying language:
[<img align="right" src="http://img.youtube.com/vi/zvzzA9GXu3Q/3.jpg" alt="zeppelin-view" hspace="10" width="200"></img>](https://www.youtube.com/watch?v=zvzzA9GXu3Q)
@ -38,7 +34,6 @@ This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Objec
This [Video Tutorial](https://www.youtube.com/watch?v=zvzzA9GXu3Q) illustrates some of the features provided by the `Geode Interpreter`.
### Create Interpreter
By default Zeppelin creates one `Geode/OQL` instance. You can remove it or create more instances.
Multiple Geode instances can be created, each configured to the same or different backend Geode cluster. But over time a `Notebook` can have only one Geode interpreter instance `bound`. That means you _cannot_ connect to different Geode clusters in the same `Notebook`. This is a known Zeppelin limitation.
@ -53,38 +48,35 @@ In the `Notebook` click on the `settings` icon in the top right corner. The sele
### Configuration
You can modify the configuration of the Geode from the `Interpreter` section. The Geode interpreter expresses the following properties:
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>geode.locator.host</td>
<td>The Geode Locator Host</td>
<td>localhost</td>
</tr>
<tr>
<td>geode.locator.port</td>
<td>The Geode Locator Port</td>
<td>10334</td>
</tr>
<tr>
<td>geode.max.result</td>
<td>Max number of OQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>geode.locator.host</td>
<td>The Geode Locator Host</td>
<td>localhost</td>
</tr>
<tr>
<td>geode.locator.port</td>
<td>The Geode Locator Port</td>
<td>10334</td>
</tr>
<tr>
<td>geode.max.result</td>
<td>Max number of OQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
### How to use
> *Tip 1: Use (CTRL + .) for OQL auto-completion.*
> *Tip 2: Always start the paragraphs with the full `%geode.oql` prefix tag! The short notation: `%geode` would still be able run the OQL queries but the syntax highlighting and the auto-completions will be disabled.*
#### Create / Destroy Regions
The OQL specification does not support [Geode Regions](https://cwiki.apache.org/confluence/display/GEODE/Index#Index-MainConceptsandComponents) mutation operations. To `create`/`destroy` regions one should use the [GFSH](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/chapter_overview.html) shell tool instead. In the following it is assumed that the GFSH is colocated with Zeppelin server.
```bash
@ -105,9 +97,7 @@ EOF
Above snippet re-creates two regions: `regionEmployee` and `regionCompany`. Note that you have to explicitly specify the locator host and port. The values should match those you have used in the Geode Interpreter configuration. Comprehensive list of [GFSH Commands by Functional Area](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/gfsh_quick_reference.html).
#### Basic OQL
#### Basic OQL
```sql
%geode.oql
SELECT count(*) FROM /regionEmployee
@ -144,12 +134,9 @@ Following query will return the EntrySet value as a Blob:
SELECT e.key, e.value FROM /regionEmployee.entrySet e
```
> Note: You can have multiple queries in the same paragraph but only the result from the first is displayed. [[1](https://issues.apache.org/jira/browse/ZEPPELIN-178)], [[2](https://issues.apache.org/jira/browse/ZEPPELIN-212)].
#### GFSH Commands From The Shell
Use the Shell Interpreter (`%sh`) to run OQL commands form the command line:
```bash
@ -159,7 +146,6 @@ gfsh -e "connect" -e "list members"
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your OQL queries. You can use both the `text input` and `select form` parameterization features
```sql

8
docs/interpreter/hive.md
View file

@ -6,12 +6,10 @@ group: manual
---
{% include JB/setup %}
## Hive Interpreter for Apache Zeppelin
The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL.
## 1. Configuration
### Configuration
<table class="table-configuration">
<tr>
<th>Property</th>
@ -71,9 +69,8 @@ The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilita
</table>
This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`.
## 2. How to use
## How to use
Basically, you can use
```sql
@ -92,7 +89,6 @@ select * from my_table;
You can also run multiple queries up to 10 by default. Changing these settings is not implemented yet.
### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parameterization features.
```sql

119
docs/interpreter/ignite.md
View file

@ -18,55 +18,54 @@ You can use Zeppelin to retrieve distributed data from cache using Ignite SQL in
### Installing and Running Ignite example
In order to use Ignite interpreters, you may install Apache Ignite in some simple steps:
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zepplin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zepplin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .<br>Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release.
2. Examples are shipped as a separate Maven project, so to start running you simply need to import provided <dest_dir>/apache-ignite-fabric-1.2.0-incubating-bin/pom.xml file into your favourite IDE, such as Eclipse.
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
* Set examples directory path to Eclipse and select the pom.xml.
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
> **Tip. If you want to run Ignite examples on the cli not IDE, you can export executable Jar file from IDE. Then run it by using below command.**
```
$ nohup java -jar </path/to/your Jar file name>
```
### Configuring Ignite Interpreter
* In case of Eclipse, Eclipse -> File -> Import -> Existing Maven Projects
* Set examples directory path to Eclipse and select the pom.xml.
* Then start `org.apache.ignite.examples.ExampleNodeStartup` (or whatever you want) to run at least one or more ignite node. When you run example code, you may notice that the number of node is increase one by one.
> **Tip. If you want to run Ignite examples on the cli not IDE, you can export executable Jar file from IDE. Then run it by using below command.**
```
$ nohup java -jar </path/to/your Jar file name>
```
### Configuring Ignite Interpreter
At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Zeppelin provides these properties for Ignite.
<table class="table-configuration">
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
</tr>
<tr>
<td>ignite.addresses</td>
<td>127.0.0.1:47500..47509</td>
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/v1.2/docs/cluster-config) section for more details.</td>
<td>ignite.addresses</td>
<td>127.0.0.1:47500..47509</td>
<td>Coma separated list of Ignite cluster hosts. See [Ignite Cluster Configuration](https://apacheignite.readme.io/v1.2/docs/cluster-config) section for more details.</td>
</tr>
<tr>
<td>ignite.clientMode</td>
<td>true</td>
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/v1.2/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
<td>ignite.clientMode</td>
<td>true</td>
<td>You can connect to the Ignite cluster as client or server node. See [Ignite Clients vs. Servers](https://apacheignite.readme.io/v1.2/docs/clients-vs-servers) section for details. Use true or false values in order to connect in client or server mode respectively.</td>
</tr>
<tr>
<td>ignite.config.url</td>
<td></td>
<td>Configuration URL. Overrides all other settings.</td>
</tr
<tr>
<td>ignite.jdbc.url</td>
<td>jdbc:ignite:cfg://default-ignite-jdbc.xml</td>
<td>Ignite JDBC connection URL.</td>
</tr>
<tr>
<td>ignite.peerClassLoadingEnabled</td>
<td>true</td>
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/v1.2/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
<td>ignite.config.url</td>
<td></td>
<td>Configuration URL. Overrides all other settings.</td>
</tr>
</table>
<tr>
<td>ignite.jdbc.url</td>
<td>jdbc:ignite:cfg://default-ignite-jdbc.xml</td>
<td>Ignite JDBC connection URL.</td>
</tr>
<tr>
<td>ignite.peerClassLoadingEnabled</td>
<td>true</td>
<td>Enables peer-class-loading. See [Zero Deployment](https://apacheignite.readme.io/v1.2/docs/zero-deployment) section for details. Use true or false values in order to enable or disable P2P class loading respectively.</td>
</tr>
</table>
![Configuration of Ignite Interpreter](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-setting.png)
@ -82,35 +81,33 @@ In order to execute SQL query, use ` %ignite.ignitesql ` prefix. <br>
Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ).
For example, you can select top 10 words in the words cache using the following query
```
%ignite.ignitesql
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
```
![IgniteSql on Zeppelin](../assets/themes/zeppelin/img/docs-img/ignite-sql-example.png)
```
%ignite.ignitesql
select _val, count(_val) as cnt from String group by _val order by cnt desc limit 10
```
![IgniteSql on Zeppelin](../assets/themes/zeppelin/img/docs-img/ignite-sql-example.png)
As long as your Ignite version and Zeppelin Ignite version is same, you can also use scala code. Please check the Zeppelin Ignite version before you download your own Ignite.
```
%ignite
import org.apache.ignite._
import org.apache.ignite.cache.affinity._
import org.apache.ignite.cache.query._
import org.apache.ignite.configuration._
```
%ignite
import org.apache.ignite._
import org.apache.ignite.cache.affinity._
import org.apache.ignite.cache.query._
import org.apache.ignite.configuration._
import scala.collection.JavaConversions._
import scala.collection.JavaConversions._
val cache: IgniteCache[AffinityUuid, String] = ignite.cache("words")
val cache: IgniteCache[AffinityUuid, String] = ignite.cache("words")
val qry = new SqlFieldsQuery("select avg(cnt), min(cnt), max(cnt) from (select count(_val) as cnt from String group by _val)", true)
val qry = new SqlFieldsQuery("select avg(cnt), min(cnt), max(cnt) from (select count(_val) as cnt from String group by _val)", true)
val res = cache.query(qry).getAll()
val res = cache.query(qry).getAll()
collectionAsScalaIterable(res).foreach(println _)
```
![Using Scala Code](../assets/themes/zeppelin/img/docs-img/ignite-scala-example.png)
collectionAsScalaIterable(res).foreach(println _)
```
![Using Scala Code](../assets/themes/zeppelin/img/docs-img/ignite-scala-example.png)
Apache Ignite also provides a guide docs for Zeppelin ["Ignite with Apache Zeppelin"](https://apacheignite.readme.io/docs/data-analysis-with-apache-zeppelin)

194
docs/interpreter/lens.md
View file

@ -16,69 +16,70 @@ group: manual
### Installing and Running Lens
In order to use Lens interpreters, you may install Apache Lens in some simple steps:
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
3. Now, you can start lens server (or stop).
```
./bin/lens-ctl start (or stop)
```
1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/).
2. Before running Lens, you have to set HIVE_HOME and HADOOP_HOME. If you want to get more information about this, please refer to [here](http://lens.apache.org/lenshome/install-and-run.html#Installation). Lens also provides Pseudo Distributed mode. [Lens pseudo-distributed setup](http://lens.apache.org/lenshome/pseudo-distributed-setup.html) is done by using [docker](https://www.docker.com/). Hive server and hadoop daemons are run as separate processes in lens pseudo-distributed setup.
3. Now, you can start lens server (or stop).
```
./bin/lens-ctl start (or stop)
```
### Configuring Lens Interpreter
At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zeppelin provides these properties for Lens.
<table class="table-configuration">
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
<th>Property Name</th>
<th>value</th>
<th>Description</th>
</tr>
<tr>
<td>lens.client.dbname</td>
<td>default</td>
<td>The database schema name</td>
<td>lens.client.dbname</td>
<td>default</td>
<td>The database schema name</td>
</tr>
<tr>
<td>lens.query.enable.persistent.resultset</td>
<td>false</td>
<td>Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions</td>
<td>lens.query.enable.persistent.resultset</td>
<td>false</td>
<td>Whether to enable persistent resultset for queries. When enabled, server will fetch results from driver, custom format them if any and store in a configured location. The file name of query output is queryhandle-id, with configured extensions</td>
</tr>
<tr>
<td>lens.server.base.url</td>
<td>http://hostname:port/lensapi</td>
<td>The base url for the lens server. you have to edit "hostname" and "port" that you may use(ex. http://0.0.0.0:9999/lensapi)</td>
<td>lens.server.base.url</td>
<td>http://hostname:port/lensapi</td>
<td>The base url for the lens server. you have to edit "hostname" and "port" that you may use(ex. http://0.0.0.0:9999/lensapi)</td>
</tr>
<tr>
<td>lens.session.cluster.user </td>
<td>default</td>
<td>Hadoop cluster username</td>
<td>lens.session.cluster.user </td>
<td>default</td>
<td>Hadoop cluster username</td>
</tr>
<tr>
<td>zeppelin.lens.maxResult</td>
<td>1000</td>
<td>Max number of rows to display</td>
<td>zeppelin.lens.maxResult</td>
<td>1000</td>
<td>Max number of rows to display</td>
</tr>
<tr>
<td>zeppelin.lens.maxThreads</td>
<td>10</td>
<td>If concurrency is true then how many threads?</td>
<td>zeppelin.lens.maxThreads</td>
<td>10</td>
<td>If concurrency is true then how many threads?</td>
</tr>
<tr>
<td>zeppelin.lens.run.concurrent</td>
<td>true</td>
<td>Run concurrent Lens Sessions</td>
<td>zeppelin.lens.run.concurrent</td>
<td>true</td>
<td>Run concurrent Lens Sessions</td>
</tr>
<tr>
<td>xxx</td>
<td>yyy</td>
<td>anything else from [Configuring lens server](https://lens.apache.org/admin/config-server.html)</td>
<td>xxx</td>
<td>yyy</td>
<td>anything else from [Configuring lens server](https://lens.apache.org/admin/config-server.html)</td>
</tr>
</table>
</table>
![Apache Lens Interpreter Setting](../assets/themes/zeppelin/img/docs-img/lens-interpreter-setting.png)
### Interpreter Bindging for Zeppelin Notebook
After configuring Lens interpreter, create your own notebook, then you can bind interpreters like below image.
After configuring Lens interpreter, create your own notebook, then you can bind interpreters like below image.
![Zeppelin Notebook Interpreter Biding](../assets/themes/zeppelin/img/docs-img/lens-interpreter-binding.png)
For more interpreter binding information see [here](http://zeppelin.incubator.apache.org/docs/manual/interpreters.html).
@ -90,84 +91,79 @@ As you can see in this video, they are using Lens Client Shell(./bin/lens-cli.sh
<li> Create and Use(Switch) Databases.
```
create database newDb
```
```
use newDb
```
```
create database newDb
```
```
use newDb
```
<li> Create Storage.
```
create storage your/path/to/lens/client/examples/resources/db-storage.xml
```
```
create storage your/path/to/lens/client/examples/resources/db-storage.xml
```
<li> Create Dimensions, Show fields and join-chains of them.
```
create dimension your/path/to/lens/client/examples/resources/customer.xml
```
```
dimension show fields customer
```
```
dimension show joinchains customer
```
```
create dimension your/path/to/lens/client/examples/resources/customer.xml
```
```
dimension show fields customer
```
```
dimension show joinchains customer
```
<li> Create Caches, Show fields and join-chains of them.
```
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
```
```
cube show fields sales
```
```
cube show joinchains sales
```
```
create cube your/path/to/lens/client/examples/resources/sales-cube.xml
```
```
cube show fields sales
```
```
cube show joinchains sales
```
<li> Create Dimtables and Fact.
```
create dimtable your/path/to/lens/client/examples/resources/customer_table.xml
```
```
create fact your/path/to/lens/client/examples/resources/sales-raw-fact.xml
```
```
create dimtable your/path/to/lens/client/examples/resources/customer_table.xml
```
```
create fact your/path/to/lens/client/examples/resources/sales-raw-fact.xml
```
<li> Add partitions to Dimtable and Fact.
```
dimtable add single-partition --dimtable_name customer_table --storage_name local --path your/path/to/lens/client/examples/resources/customer-local-part.xml
```
```
fact add partitions --fact_name sales_raw_fact --storage_name local --path your/path/to/lens/client/examples/resources/sales-raw-local-parts.xml
```
```
dimtable add single-partition --dimtable_name customer_table --storage_name local --path your/path/to/lens/client/examples/resources/customer-local-part.xml
```
```
fact add partitions --fact_name sales_raw_fact --storage_name local --path your/path/to/lens/client/examples/resources/sales-raw-local-parts.xml
```
<li> Now, you can run queries on cubes.
```
query execute cube select customer_city_name, product_details.description, product_details.category, product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-11-00', '2015-04-13-00')
```
![Lens Query Result](../assets/themes/zeppelin/img/docs-img/lens-result.png)
```
query execute cube select customer_city_name, product_details.description, product_details.category, product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-11-00', '2015-04-13-00')
```
![Lens Query Result](../assets/themes/zeppelin/img/docs-img/lens-result.png)
These are just examples that provided in advance by Lens. If you want to explore whole tutorials of Lens, see the [tutorial video](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples).
### Lens UI Service
Lens also provides web UI service. Once the server starts up, you can open the service on http://serverhost:19999/index.html and browse. You may also check the structure that you made and use query easily here.
![Lens UI Servive](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png)
![Lens UI Servive](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png)

81
docs/interpreter/postgresql.md
View file

@ -6,10 +6,7 @@ group: manual
---
{% include JB/setup %}
## PostgreSQL, HAWQ Interpreter for Apache Zeppelin
<br/>
<table class="table-configuration">
<tr>
<th>Name</th>
@ -23,7 +20,6 @@ group: manual
</tr>
</table>
<br/>
[<img align="right" src="http://img.youtube.com/vi/wqXXQhJ5Uk8/0.jpg" alt="zeppelin-view" hspace="10" width="250"></img>](https://www.youtube.com/watch?v=wqXXQhJ5Uk8)
This interpreter seamlessly supports the following SQL data processing engines:
@ -32,11 +28,9 @@ This interpreter seamlessly supports the following SQL data processing engines:
* [Apache HAWQ](http://pivotal.io/big-data/pivotal-hawq) - Powerful [Open Source](https://wiki.apache.org/incubator/HAWQProposal) SQL-On-Hadoop engine.
* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL.
This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`.
### Create Interpreter
By default Zeppelin creates one `PSQL` instance. You can remove it or create new instances.
Multiple PSQL instances can be created, each configured to the same or different backend databases. But over time a `Notebook` can have only one PSQL interpreter instance `bound`. That means you _cannot_ connect to different databases in the same `Notebook`. This is a known Zeppelin limitation.
@ -51,47 +45,45 @@ In the `Notebook` click on the `settings` icon in the top right corner. The sele
### Configuration
You can modify the configuration of the PSQL from the `Interpreter` section. The PSQL interpreter expenses the following properties:
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>postgresql.url</td>
<td>JDBC URL to connect to </td>
<td>jdbc:postgresql://localhost:5432</td>
</tr>
<tr>
<td>postgresql.user</td>
<td>JDBC user name</td>
<td>gpadmin</td>
</tr>
<tr>
<td>postgresql.password</td>
<td>JDBC password</td>
<td></td>
</tr>
<tr>
<td>postgresql.driver.name</td>
<td>JDBC driver name. In this version the driver name is fixed and should not be changed</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>postgresql.max.result</td>
<td>Max number of SQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
<table class="table-configuration">
<tr>
<th>Property Name</th>
<th>Description</th>
<th>Default Value</th>
</tr>
<tr>
<td>postgresql.url</td>
<td>JDBC URL to connect to </td>
<td>jdbc:postgresql://localhost:5432</td>
</tr>
<tr>
<td>postgresql.user</td>
<td>JDBC user name</td>
<td>gpadmin</td>
</tr>
<tr>
<td>postgresql.password</td>
<td>JDBC password</td>
<td></td>
</tr>
<tr>
<td>postgresql.driver.name</td>
<td>JDBC driver name. In this version the driver name is fixed and should not be changed</td>
<td>org.postgresql.Driver</td>
</tr>
<tr>
<td>postgresql.max.result</td>
<td>Max number of SQL result to display to prevent the browser overload</td>
<td>1000</td>
</tr>
</table>
### How to use
```
Tip: Use (CTRL + .) for SQL auto-completion.
```
#### DDL and SQL commands
#### DDL and SQL commands
Start the paragraphs with the full `%psql.sql` prefix tag! The short notation: `%psql` would still be able run the queries but the syntax highlighting and the auto-completions will be disabled.
You can use the standard CREATE / DROP / INSERT commands to create or modify the data model:
@ -121,7 +113,6 @@ select * from mytable;
```
#### PSQL command line tools
Use the Shell Interpreter (`%sh`) to access the command line [PSQL](http://www.postgresql.org/docs/9.4/static/app-psql.html) interactively:
```bash
@ -131,6 +122,7 @@ psql -h phd3.localdomain -U gpadmin -p 5432 <<EOF
\q
EOF
```
This will produce output like this:
```
@ -146,7 +138,6 @@ This will produce output like this:
```
#### Apply Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features
```sql
@ -157,8 +148,8 @@ GROUP BY ${group_by=product_id,product_id|product_name|customer_id|store_id}
ORDER BY count ${order=DESC,DESC|ASC}
LIMIT ${limit=10};
```
#### Example HAWQ PXF/HDFS Tables
#### Example HAWQ PXF/HDFS Tables
Create HAWQ external table that read data from tab-separated-value data in HDFS.
```sql
@ -168,11 +159,13 @@ CREATE EXTERNAL TABLE retail_demo.payment_methods_pxf (
payment_method_code character varying(20)
) LOCATION ('pxf://${NAME_NODE_HOST}:50070/retail_demo/payment_methods.tsv.gz?profile=HdfsTextSimple') FORMAT 'TEXT' (DELIMITER = E'\t');
```
And retrieve content
```sql
%psql.sql
select * from retail_demo.payment_methods_pxf
```
### Auto-completion
The PSQL Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. In addition to the SQL keyword the interpreter provides suggestions for the Schema, Table, Column names as well.

17
docs/interpreter/scalding.md
View file

@ -6,7 +6,6 @@ group: manual
---
{% include JB/setup %}
## Scalding Interpreter for Apache Zeppelin
[Scalding](https://github.com/twitter/scalding) is an open source Scala library for writing MapReduce jobs.
@ -18,20 +17,20 @@ mvn clean package -Pscalding -DskipTests
```
### Enabling the Scalding Interpreter
In a notebook, to enable the **Scalding** interpreter, click on the **Gear** icon,select **Scalding**, and hit **Save**.
<center>
![Interpreter Binding](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterBinding.png)
![Interpreter Selection](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterSelection.png)
</center>
<center>
![Interpreter Binding](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterBinding.png)
![Interpreter Selection](../assets/themes/zeppelin/img/docs-img/scalding-InterpreterSelection.png)
</center>
### Configuring the Interpreter
Zeppelin comes with a pre-configured Scalding interpreter in local mode, so you do not need to install anything.
### Testing the Interpreter
In example, by using the [Alice in Wonderland](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) tutorial, we will count words (of course!), and plot a graph of the top 10 words in the book.
```
@ -75,4 +74,4 @@ If you click on the icon for the pie chart, you should be able to see a chart li
### Current Status & Future Work
The current implementation of the Scalding interpreter does not support canceling jobs, or fine-grained progress updates.
The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).
The pre-configured Scalding interpreter only supports Scalding in local mode. Hadoop mode for Scalding is currently unsupported, and will be future work (contributions welcome!).

17
docs/interpreter/spark.md
View file

@ -8,7 +8,6 @@ group: manual
## Spark Interpreter for Apache Zeppelin
[Apache Spark](http://spark.apache.org) is supported in Zeppelin with
Spark Interpreter group, which consisted of 4 interpreters.
@ -40,14 +39,10 @@ Spark Interpreter group, which consisted of 4 interpreters.
</tr>
</table>
<br />
## Configuration
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
### 1. Export SPARK_HOME
In **conf/zeppelin-env.sh**, export `SPARK_HOME` environment variable with your Spark installation path.
for example
@ -64,7 +59,6 @@ export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"
```
### 2. Set master in Interpreter menu
After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type.
for example,
@ -74,25 +68,21 @@ for example,
* **yarn-client** in Yarn client mode
* **mesos://host:5050** in Mesos cluster
That's it. Zeppelin will work with any version of Spark and any deployment type without rebuilding Zeppelin in this way. ( Zeppelin 0.5.5-incubating release works up to Spark 1.5.2 )
> Note that without exporting `SPARK_HOME`, it's running in local mode with included version of Spark. The included version may vary depending on the build profile.
<br />
## SparkContext, SQLContext, ZeppelinContext
SparkContext, SQLContext, ZeppelinContext are automatically created and exposed as variable names 'sc', 'sqlContext' and 'z', respectively, both in scala and python environments.
> Note that scala / python environment shares the same SparkContext, SQLContext, ZeppelinContext instance.
<br />
<a name="dependencyloading"> </a>
## Dependency Management
There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties.
### 1. Dynamic Dependency Loading via %dep interpreter
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter.
* Load libraries recursively from Maven repository
@ -182,15 +172,13 @@ Here are few examples:
spark.jars.packages com.databricks:spark-csv_2.10:1.2.0
spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip
<br />
## ZeppelinContext
Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility.
### Object Exchange
ZeppelinContext extends map and it's shared between scala, python environment.
So you can put some object from scala and read it from python, vise versa.
<div class="codetabs">
<div data-lang="scala" markdown="1">
@ -212,6 +200,7 @@ myObject = z.get("objName")
</div>
</div>
### Form Creation
ZeppelinContext provides functions for creating forms.

216
docs/interpreter/tachyon.md Normal file
View file

@ -0,0 +1,216 @@
---
layout: page
title: "Tachyon Interpreter"
description: "Tachyon Interpreter"
group: manual
---
{% include JB/setup %}
## Tachyon Interpreter for Apache Zeppelin
[Tachyon](http://tachyon-project.org/) is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks.
## Configuration
<table class="table-configuration">
<tr>
<th>Name</th>
<th>Class</th>
<th>Description</th>
</tr>
<tr>
<td>tachyon.master.hostname</td>
<td>localhost</td>
<td>Tachyon master hostname</td>
</tr>
<tr>
<td>tachyon.master.port</td>
<td>19998</td>
<td>Tachyon master port</td>
</tr>
</table>
## Enabling Tachyon Interpreter
In a notebook, to enable the **Tachyon** interpreter, click on the **Gear** icon and select **Tachyon**.
## Using the Tachyon Interpreter
In a paragraph, use `%tachyon` to select the **Tachyon** interpreter and then input all commands.
```bash
%tachyon
help
```
> **Tip :** Use ( Ctrl + . ) for autocompletion.
## Interpreter Commands
The **Tachyon** interpreter accepts the following commands.
<center>
<table class="table-configuration">
<tr>
<th>Operation</th>
<th>Syntax</th>
<th>Description</th>
</tr>
<tr>
<td>copyFromLocal</td>
<td>copyFromLocal "source path" "remote path"</td>
<td>Copy the specified file specified by "source path" to the path specified by "remote path".
This command will fail if "remote path" already exists.</td>
</tr>
<tr>
<td>copyToLocal</td>
<td>copyToLocal "remote path" "local path"</td>
<td>Copy the specified file from the path specified by "remote source" to a local
destination.</td>
</tr>
<tr>
<td>count</td>
<td>count "path"</td>
<td>Display the number of folders and files matching the specified prefix in "path".</td>
</tr>
<tr>
<td>du</td>
<td>du "path"</td>
<td>Display the size of a file or a directory specified by the input path.</td>
</tr>
<tr>
<td>fileinfo</td>
<td>fileinfo "path"</td>
<td>Print the information of the blocks of a specified file.</td>
</tr>
<tr>
<td>free</td>
<td>free "path"</td>
<td>Free a file or all files under a directory from Tachyon. If the file/directory is also
in under storage, it will still be available there.</td>
</tr>
<tr>
<td>getCapacityBytes</td>
<td>getCapacityBytes</td>
<td>Get the capacity of the TachyonFS.</td>
</tr>
<tr>
<td>getUsedBytes</td>
<td>getUsedBytes</td>
<td>Get number of bytes used in the TachyonFS.</td>
</tr>
<tr>
<td>load</td>
<td>load "path"</td>
<td>Load the data of a file or a directory from under storage into Tachyon.</td>
</tr>
<tr>
<td>loadMetadata</td>
<td>loadMetadata "path"</td>
<td>Load the metadata of a file or a directory from under storage into Tachyon.</td>
</tr>
<tr>
<td>location</td>
<td>location "path"</td>
<td>Display a list of hosts that have the file data.</td>
</tr>
<tr>
<td>ls</td>
<td>ls "path"</td>
<td>List all the files and directories directly under the given path with information such as
size.</td>
</tr>
<tr>
<td>lsr</td>
<td>lsr "path"</td>
<td>Recursively list all the files and directories under the given path with information such
as size.</td>
</tr>
<tr>
<td>mkdir</td>
<td>mkdir "path"</td>
<td>Create a directory under the given path, along with any necessary parent directories. This
command will fail if the given path already exists.</td>
</tr>
<tr>
<td>mount</td>
<td>mount "path" "uri"</td>
<td>Mount the underlying file system path "uri" into the Tachyon namespace as "path". The "path"
is assumed not to exist and is created by the operation. No data or metadata is loaded from under
storage into Tachyon. After a path is mounted, operations on objects under the mounted path are
mirror to the mounted under storage.</td>
</tr>
<tr>
<td>mv</td>
<td>mv "source" "destination"</td>
<td>Move a file or directory specified by "source" to a new location "destination". This command
will fail if "destination" already exists.</td>
</tr>
<tr>
<td>pin</td>
<td>pin "path"</td>
<td>Pin the given file to avoid evicting it from memory. If the given path is a directory, it
recursively pins all the files contained and any new files created within this directory.</td>
</tr>
<tr>
<td>report</td>
<td>report "path"</td>
<td>Report to the master that a file is lost.</td>
</tr>
<tr>
<td>request</td>
<td>request "path" "dependency ID"</td>
<td>Request the file for a given dependency ID.</td>
</tr>
<tr>
<td>rm</td>
<td>rm "path"</td>
<td>Remove a file. This command will fail if the given path is a directory rather than a
file.</td>
</tr>
<tr>
<td>rmr</td>
<td>rmr "path"</td>
<td>Remove a file, or a directory with all the files and sub-directories that this directory
contains.</td>
</tr>
<tr>
<td>tail</td>
<td>tail "path"</td>
<td>Print the last 1KB of the specified file to the console.</td>
</tr>
<tr>
<td>touch</td>
<td>touch "path"</td>
<td>Create a 0-byte file at the specified location.</td>
</tr>
<tr>
<td>unmount</td>
<td>unmount "path"</td>
<td>Unmount the underlying file system path mounted in the Tachyon namespace as "path". Tachyon
objects under "path" are removed from Tachyon, but they still exist in the previously mounted
under storage.</td>
</tr>
<tr>
<td>unpin</td>
<td>unpin "path"</td>
<td>Unpin the given file to allow Tachyon to evict this file again. If the given path is a
directory, it recursively unpins all files contained and any new files created within this
directory.</td>
</tr>
</table>
</center>
## How to test it's working
Be sure to have configured correctly the Tachyon interpreter, then open a new paragraph and type one of the above commands.
Below a simple example to show how to interact with Tachyon interpreter.
Following steps are performed:
* using sh interpreter a new text file is created on local machine
* using Tachyon interpreter:
* is listed the content of the tfs (Tachyon File System) root
* the file previously created is copied to tfs
* is listed again the content of the tfs root to check the existence of the new copied file
* is showed the content of the copied file (using the tail command)
* the file previously copied to tfs is copied to local machine
* using sh interpreter it's checked the existence of the new file copied from Tachyon and its content is showed
<center>
![Tachyon Interpreter Example](../assets/themes/zeppelin/img/docs-img/tachyon-example.png)
</center>

11
docs/manual/interpreters.md
View file

@ -19,31 +19,24 @@ limitations under the License.
-->
{% include JB/setup %}
## Interpreters in Zeppelin
In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin.
The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin.
Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, Hive, Markdown, Shell and so on.
<br/>
## What is Zeppelin interpreter?
Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific language/data-processing-backend. For example, to use scala code in Zeppelin, you need `%spark` interpreter.
When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_create.png">
<br/>
## What is Zeppelin Interpreter Setting?
Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
<br/>
## What is Zeppelin Interpreter Group?
Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter.
By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, spark interpreter group is including Spark support, pySpark,
SparkSQL and the dependency loader.
@ -53,9 +46,7 @@ Technically, Zeppelin interpreters from the same group are running in the same J
Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image.
<img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting_spark.png">
<br/>
## Programming Languages for Interpreter
If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor.
To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict).
@ -65,5 +56,3 @@ If you want to add a new set of syntax highlighting,
1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ).
2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name.
3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`.

143
docs/rest-api/rest-configuration.md Normal file
View file

@ -0,0 +1,143 @@
---
layout: page
title: "Configuration REST API"
description: ""
group: rest-api
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
## Zeppelin REST API
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
All REST API are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
[JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
If you work with zeppelin and find a need for an additional REST API please [file an issue or send us mail](../../community.html)
<br />
### Configuration REST API list
<table class="table-configuration">
<col width="200">
<tr>
<th>List configurations</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```GET``` method return all key/value pair of configurations on the server.<br/>
Note: For security reason, some pairs would not be shown.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/configurations/all```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>
<pre>
{
"status":"OK",
"message":"",
"body":{
"zeppelin.war.tempdir":"webapps",
"zeppelin.notebook.homescreen.hide":"false",
"zeppelin.interpreter.remoterunner":"bin/interpreter.sh",
"zeppelin.notebook.s3.user":"user",
"zeppelin.server.port":"8089",
"zeppelin.dep.localrepo":"local-repo",
"zeppelin.ssl.truststore.type":"JKS",
"zeppelin.ssl.keystore.path":"keystore",
"zeppelin.notebook.s3.bucket":"zeppelin",
"zeppelin.server.addr":"0.0.0.0",
"zeppelin.ssl.client.auth":"false",
"zeppelin.server.context.path":"/",
"zeppelin.ssl.keystore.type":"JKS",
"zeppelin.ssl.truststore.path":"truststore",
"zeppelin.interpreters":"org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter",
"zeppelin.ssl":"false",
"zeppelin.notebook.autoInterpreterBinding":"true",
"zeppelin.notebook.homescreen":"",
"zeppelin.notebook.storage":"org.apache.zeppelin.notebook.repo.VFSNotebookRepo",
"zeppelin.interpreter.connect.timeout":"30000",
"zeppelin.anonymous.allowed":"true",
"zeppelin.server.allowed.origins":"*",
"zeppelin.encoding":"UTF-8"
}
}
</pre>
</td>
</tr>
</table>
<br/>
<table class="table-configuration">
<col width="200">
<tr>
<th>List configurations (prefix match)</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```GET``` method return all prefix matched key/value pair of configurations on the server.<br/>
Note: For security reason, some pairs would not be shown.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/configurations/prefix/[prefix]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td> sample JSON response
</td>
<td>
<pre>
{
"status":"OK",
"message":"",
"body":{
"zeppelin.ssl.keystore.type":"JKS",
"zeppelin.ssl.truststore.path":"truststore",
"zeppelin.ssl.truststore.type":"JKS",
"zeppelin.ssl.keystore.path":"keystore",
"zeppelin.ssl":"false",
"zeppelin.ssl.client.auth":"false"
}
}
</pre>
</td>
</tr>
</table>

3
docs/rest-api/rest-interpreter.md
View file

@ -23,10 +23,9 @@ limitations under the License.
Zeppelin provides several REST API's for interaction and remote activation of zeppelin functionality.
All REST API are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`.
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON view such as
Note that zeppein REST API receive or return JSON objects, it it recommended you install some JSON viewers such as
[JSON View](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc).
If you work with zeppelin and find a need for an additional REST API, please [file an issue or send us mail](http://zeppelin.incubator.apache.org/community.html).
<br />

113
docs/rest-api/rest-notebook.md
View file

@ -24,7 +24,7 @@ limitations under the License.
All REST APIs are available starting with the following endpoint ```http://[zeppelin-server]:[zeppelin-port]/api```
Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewer
Note that zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers
such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc)
@ -33,7 +33,7 @@ limitations under the License.
<br />
### Notebook REST API list
Notebooks REST API supports the following operations: List, Create, Get, Delete, Clone, Run as detailed in the following table
Notebooks REST API supports the following operations: List, Create, Get, Delete, Clone, Run, Export, Import as detailed in the following table
<table class="table-configuration">
<col width="200">
@ -773,3 +773,112 @@ limitations under the License.
</tr>
</table>
<table class="table-configuration">
<col width="200">
<tr>
<th>Export notebook</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```GET``` method exports a notebook by the given id and gernerates a JSON
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/export/[notebookId]```</td>
</tr>
<tr>
<td>Success code</td>
<td>201</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<td> sample JSON response </td>
<td><pre>{
"paragraphs": [
{
"text": "%md This is my new paragraph in my new note",
"dateUpdated": "Jan 8, 2016 4:49:38 PM",
"config": {
"enabled": true
},
"settings": {
"params": {},
"forms": {}
},
"jobName": "paragraph_1452300578795_1196072540",
"id": "20160108-164938_1685162144",
"dateCreated": "Jan 8, 2016 4:49:38 PM",
"status": "READY",
"progressUpdateIntervalMs": 500
}
],
"name": "source note for export",
"id": "2B82H3RR1",
"angularObjects": {},
"config": {},
"info": {}
}</pre></td>
</tr>
</table>
<table class="table-configuration">
<col width="200">
<tr>
<th>Export notebook</th>
<th></th>
</tr>
<tr>
<td>Description</td>
<td>This ```POST``` method imports a notebook from the notebook JSON input
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/import```</td>
</tr>
<tr>
<td>Success code</td>
<td>201</td>
</tr>
<tr>
<td> Fail code</td>
<td> 500 </td>
</tr>
<td> sample JSON input </td>
<td><pre>{
"paragraphs": [
{
"text": "%md This is my new paragraph in my new note",
"dateUpdated": "Jan 8, 2016 4:49:38 PM",
"config": {
"enabled": true
},
"settings": {
"params": {},
"forms": {}
},
"jobName": "paragraph_1452300578795_1196072540",
"id": "20160108-164938_1685162144",
"dateCreated": "Jan 8, 2016 4:49:38 PM",
"status": "READY",
"progressUpdateIntervalMs": 500
}
],
"name": "source note for export",
"id": "2B82H3RR1",
"angularObjects": {},
"config": {},
"info": {}
}</pre></td>
<tr>
<td> sample JSON response </td>
<td><pre>"status": "CREATED","message": "","body": "2AZPHY918"}</pre></td>
</tr>
</tr>
</table>

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class FlinkInterpreterTest {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null);
context = new InterpreterContext(null, null, null, null, null, null, null, null, null);
}
@AfterClass

12
hive/src/test/java/org/apache/zeppelin/hive/HiveInterpreterTest.java
View file

@ -79,9 +79,9 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null)).message().contains("SCHEMA_NAME"));
assertTrue(t.interpret("show databases", new InterpreterContext("", "1", "","", null,null,null,null,null)).message().contains("SCHEMA_NAME"));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null)).message());
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
}
@Test
@ -101,7 +101,7 @@ public class HiveInterpreterTest {
t.open();
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n",
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null)).message());
t.interpret("(h2)\n select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null)).message());
}
@Test
@ -117,13 +117,13 @@ public class HiveInterpreterTest {
t.open();
InterpreterResult interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
t.getConnection("default").close();
interpreterResult =
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null));
t.interpret("select * from test_table", new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\n", interpreterResult.message());
}
@ -139,7 +139,7 @@ public class HiveInterpreterTest {
HiveInterpreter t = new HiveInterpreter(properties);
t.open();
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null);
InterpreterContext interpreterContext = new InterpreterContext(null, "a", null, null, null, null, null, null, null);
//simple select test
InterpreterResult result = t.interpret("select * from test_table", interpreterContext);

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -44,7 +44,7 @@ public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;

4
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -94,7 +94,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
@ -116,7 +116,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null));
InterpreterResult interpreterResult = t.interpret(sqlQuery, new InterpreterContext("", "1", "","", null,null,null,null,null));
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());

10
notebook/2A94M5J1Z/note.json
View file

@ -35,6 +35,7 @@
{
"title": "Load data into table",
"text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")",
"dateUpdated": "Jan 14, 2016 7:58:56 PM",
"config": {
"colWidth": 12.0,
"graph": {
@ -46,7 +47,9 @@
"groups": [],
"scatter": {}
},
"title": true
"title": true,
"enabled": true,
"editorMode": "ace/mode/scala"
},
"settings": {
"params": {},
@ -333,7 +336,10 @@
],
"name": "Zeppelin Tutorial",
"id": "2A94M5J1Z",
"angularObjects": {},
"angularObjects": {
"2B6FF8NNU": [],
"2B67PH63Z": []
},
"config": {
"looknfeel": "default"
},

1
pom.xml
View file

@ -103,6 +103,7 @@
<module>lens</module>
<module>cassandra</module>
<module>elasticsearch</module>
<module>tachyon</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-distribution</module>

2
scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java
View file

@ -65,7 +65,7 @@ public class ScaldingInterpreterTest {
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), null);
}
@After

59
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -19,18 +19,22 @@ package org.apache.zeppelin.shell;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
@ -41,6 +45,7 @@ import org.slf4j.LoggerFactory;
*/
public class ShellInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(ShellInterpreter.class);
private static final String EXECUTOR_KEY = "executor";
int commandTimeOut = 600000;
static {
@ -61,30 +66,66 @@ public class ShellInterpreter extends Interpreter {
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.debug("Run shell command '" + cmd + "'");
long start = System.currentTimeMillis();
CommandLine cmdLine = CommandLine.parse("bash");
cmdLine.addArgument("-c", false);
cmdLine.addArgument(cmd, false);
DefaultExecutor executor = new DefaultExecutor();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(outputStream));
ByteArrayOutputStream errorStream = new ByteArrayOutputStream();
executor.setStreamHandler(new PumpStreamHandler(outputStream, errorStream));
executor.setWatchdog(new ExecuteWatchdog(commandTimeOut));
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
Map<String, Object> info = runningJob.info();
info.put(EXECUTOR_KEY, executor);
try {
int exitValue = executor.execute(cmdLine);
int exitVal = executor.execute(cmdLine);
logger.info("Paragraph " + contextInterpreter.getParagraphId()
+ "return with exit value: " + exitVal);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputStream.toString());
} catch (ExecuteException e) {
int exitValue = e.getExitValue();
logger.error("Can not run " + cmd, e);
return new InterpreterResult(Code.ERROR, e.getMessage());
Code code = Code.ERROR;
String msg = errorStream.toString();
if (exitValue == 143) {
code = Code.INCOMPLETE;
msg = msg + "Paragraph received a SIGTERM.\n";
logger.info("The paragraph " + contextInterpreter.getParagraphId()
+ " stopped executing: " + msg);
}
msg += "Exitvalue: " + exitValue;
return new InterpreterResult(code, msg);
} catch (IOException e) {
logger.error("Can not run " + cmd, e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {}
private Job getRunningJob(String paragraphId) {
Job foundJob = null;
Collection<Job> jobsRunning = getScheduler().getJobsRunning();
for (Job job : jobsRunning) {
if (job.getId().equals(paragraphId)) {
foundJob = job;
}
}
return foundJob;
}
@Override
public void cancel(InterpreterContext context) {
Job runningJob = getRunningJob(context.getParagraphId());
if (runningJob != null) {
Map<String, Object> info = runningJob.info();
Object object = info.get(EXECUTOR_KEY);
if (object != null) {
Executor executor = (Executor) object;
ExecuteWatchdog watchdog = executor.getWatchdog();
watchdog.destroyProcess();
}
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
@ -97,8 +138,8 @@ public class ShellInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
ShellInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton().createOrGetParallelScheduler(
ShellInterpreter.class.getName() + this.hashCode(), 10);
}
@Override

36
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -73,8 +73,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
private GatewayServer gatewayServer;
private DefaultExecutor executor;
private int port;
private ByteArrayOutputStream outputStream;
private ByteArrayOutputStream errStream;
private SparkOutputStream outputStream;
private BufferedWriter ins;
private PipedInputStream in;
private ByteArrayOutputStream input;
@ -173,7 +172,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
cmd.addArgument(Integer.toString(port), false);
cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
executor = new DefaultExecutor();
outputStream = new ByteArrayOutputStream();
outputStream = new SparkOutputStream();
PipedOutputStream ps = new PipedOutputStream();
in = null;
try {
@ -274,7 +273,6 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
statementError = error;
statementFinishedNotifier.notify();
}
}
boolean pythonScriptInitialized = false;
@ -287,6 +285,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
public void appendOutput(String message) throws IOException {
outputStream.getInterpreterOutput().write(message);
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
SparkInterpreter sparkInterpreter = getSparkInterpreter();
@ -300,7 +302,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
+ outputStream.toString());
}
outputStream.reset();
outputStream.setInterpreterOutput(context.out);
synchronized (pythonScriptInitializeNotifier) {
long startTime = System.currentTimeMillis();
@ -314,15 +316,24 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
String errorMessage = "";
try {
context.out.flush();
errorMessage = new String(context.out.toByteArray());
} catch (IOException e) {
throw new InterpreterException(e);
}
if (pythonscriptRunning == false) {
// python script failed to initialize and terminated
return new InterpreterResult(Code.ERROR, "failed to start pyspark"
+ outputStream.toString());
+ errorMessage);
}
if (pythonScriptInitialized == false) {
// timeout. didn't get initialized message
return new InterpreterResult(Code.ERROR, "pyspark is not responding "
+ outputStream.toString());
+ errorMessage);
}
if (!sparkInterpreter.getSparkVersion().isPysparkSupported()) {
@ -352,7 +363,14 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
if (statementError) {
return new InterpreterResult(Code.ERROR, statementOutput);
} else {
return new InterpreterResult(Code.SUCCESS, statementOutput);
try {
context.out.flush();
} catch (IOException e) {
throw new InterpreterException(e);
}
return new InterpreterResult(Code.SUCCESS);
}
}
@ -389,8 +407,6 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
return new LinkedList<String>();
}
outputStream.reset();
pythonInterpretRequest = new PythonInterpretRequest(completionCommand, "");
statementOutput = null;

32
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -17,9 +17,7 @@
package org.apache.zeppelin.spark;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
@ -41,7 +39,6 @@ import org.apache.spark.repl.SparkJLineCompletion;
import org.apache.spark.scheduler.ActiveJob;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.Pool;
import org.apache.spark.scheduler.SparkListener;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.ui.jobs.JobProgressListener;
import org.apache.zeppelin.interpreter.Interpreter;
@ -115,7 +112,7 @@ public class SparkInterpreter extends Interpreter {
private SparkILoop interpreter;
private SparkIMain intp;
private SparkContext sc;
private ByteArrayOutputStream out;
private SparkOutputStream out;
private SQLContext sqlc;
private SparkDependencyResolver dep;
private SparkJLineCompletion completor;
@ -129,7 +126,7 @@ public class SparkInterpreter extends Interpreter {
public SparkInterpreter(Properties property) {
super(property);
out = new ByteArrayOutputStream();
out = new SparkOutputStream();
}
public SparkInterpreter(Properties property, SparkContext sc) {
@ -452,10 +449,9 @@ public class SparkInterpreter extends Interpreter {
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
PrintStream printStream = new PrintStream(out);
/* spark interpreter */
this.interpreter = new SparkILoop(null, new PrintWriter(out));
interpreter.settings_$eq(settings);
interpreter.createInterpreter();
@ -481,7 +477,7 @@ public class SparkInterpreter extends Interpreter {
dep = getDependencyResolver();
z = new ZeppelinContext(sc, sqlc, null, dep, printStream,
z = new ZeppelinContext(sc, sqlc, null, dep,
Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
@ -489,7 +485,6 @@ public class SparkInterpreter extends Interpreter {
binder.put("sc", sc);
binder.put("sqlc", sqlc);
binder.put("z", z);
binder.put("out", printStream);
intp.interpret("@transient val z = "
+ "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]");
@ -675,13 +670,13 @@ public class SparkInterpreter extends Interpreter {
synchronized (this) {
z.setGui(context.getGui());
sc.setJobGroup(getJobGroup(context), "Zeppelin", false);
InterpreterResult r = interpretInput(lines);
InterpreterResult r = interpretInput(lines, context);
sc.clearJobGroup();
return r;
}
}
public InterpreterResult interpretInput(String[] lines) {
public InterpreterResult interpretInput(String[] lines, InterpreterContext context) {
SparkEnv.set(env);
// add print("") to make sure not finishing with comment
@ -692,8 +687,9 @@ public class SparkInterpreter extends Interpreter {
}
linesToRun[lines.length] = "print(\"\")";
Console.setOut((java.io.PrintStream) binder.get("out"));
out.reset();
Console.setOut(context.out);
out.setInterpreterOutput(context.out);
context.out.clear();
Code r = null;
String incomplete = "";
@ -713,6 +709,7 @@ public class SparkInterpreter extends Interpreter {
res = intp.interpret(incomplete + s);
} catch (Exception e) {
sc.clearJobGroup();
out.setInterpreterOutput(null);
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
@ -721,7 +718,8 @@ public class SparkInterpreter extends Interpreter {
if (r == Code.ERROR) {
sc.clearJobGroup();
return new InterpreterResult(r, out.toString());
out.setInterpreterOutput(null);
return new InterpreterResult(r, "");
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
@ -730,9 +728,13 @@ public class SparkInterpreter extends Interpreter {
}
if (r == Code.INCOMPLETE) {
sc.clearJobGroup();
out.setInterpreterOutput(null);
return new InterpreterResult(r, "Incomplete expression");
} else {
return new InterpreterResult(r, out.toString());
sc.clearJobGroup();
out.setInterpreterOutput(null);
return new InterpreterResult(Code.SUCCESS);
}
}

75
spark/src/main/java/org/apache/zeppelin/spark/SparkOutputStream.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import java.io.IOException;
import java.io.OutputStream;
/**
* InterpreterOutput can be attached / detached.
*/
public class SparkOutputStream extends OutputStream {
InterpreterOutput interpreterOutput;
public SparkOutputStream() {
}
public InterpreterOutput getInterpreterOutput() {
return interpreterOutput;
}
public void setInterpreterOutput(InterpreterOutput interpreterOutput) {
this.interpreterOutput = interpreterOutput;
}
@Override
public void write(int b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte [] b) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b);
}
}
@Override
public void write(byte [] b, int offset, int len) throws IOException {
if (interpreterOutput != null) {
interpreterOutput.write(b, offset, len);
}
}
@Override
public void close() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.close();
}
}
@Override
public void flush() throws IOException {
if (interpreterOutput != null) {
interpreterOutput.flush();
}
}
}

73
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -21,6 +21,7 @@ import static scala.collection.JavaConversions.asJavaCollection;
import static scala.collection.JavaConversions.asJavaIterable;
import static scala.collection.JavaConversions.collectionAsScalaIterable;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -54,19 +55,17 @@ import scala.collection.Iterable;
*/
public class ZeppelinContext extends HashMap<String, Object> {
private SparkDependencyResolver dep;
private PrintStream out;
private InterpreterContext interpreterContext;
private int maxResult;
public ZeppelinContext(SparkContext sc, SQLContext sql,
InterpreterContext interpreterContext,
SparkDependencyResolver dep, PrintStream printStream,
SparkDependencyResolver dep,
int maxResult) {
this.sc = sc;
this.sqlContext = sql;
this.interpreterContext = interpreterContext;
this.dep = dep;
this.out = printStream;
this.maxResult = maxResult;
}
@ -273,10 +272,15 @@ public class ZeppelinContext extends HashMap<String, Object> {
throw new InterpreterException("Can not road DataFrame/SchemaRDD class");
}
if (cls.isInstance(o)) {
out.print(showDF(sc, interpreterContext, o, maxResult));
} else {
out.print(o.toString());
try {
if (cls.isInstance(o)) {
interpreterContext.out.write(showDF(sc, interpreterContext, o, maxResult));
} else {
interpreterContext.out.write(o.toString());
}
} catch (IOException e) {
throw new InterpreterException(e);
}
}
@ -423,7 +427,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
/**
* Run paragraphs
* @param paragraphIdOrIdxs list of paragraph id or idx
* @param paragraphIdOrIdx list of paragraph id or idx
*/
public void run(List<Object> paragraphIdOrIdx, InterpreterContext context) {
for (Object idOrIdx : paragraphIdOrIdx) {
@ -471,17 +475,17 @@ public class ZeppelinContext extends HashMap<String, Object> {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
String noteId = interpreterContext.getNoteId();
// try get local object
AngularObject ao = registry.get(name, interpreterContext.getNoteId());
AngularObject ao = registry.get(name, interpreterContext.getNoteId(), null);
if (ao == null) {
// then global object
ao = registry.get(name, null);
ao = registry.get(name, null, null);
}
return ao;
}
/**
* Get angular object. Look up local registry first and then global registry
* Get angular object. Look up notebook scope first and then global scope
* @param name variable name
* @return value
*/
@ -495,13 +499,13 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Get angular object. Look up global registry
* Get angular object. Look up global scope
* @param name variable name
* @return value
*/
public Object angularGlobal(String name) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
AngularObject ao = registry.get(name, null);
AngularObject ao = registry.get(name, null, null);
if (ao == null) {
return null;
} else {
@ -510,7 +514,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Create angular variable in local registry and bind with front end Angular display system.
* Create angular variable in notebook scope and bind with front end Angular display system.
* If variable exists, it'll be overwritten.
* @param name name of the variable
* @param o value
@ -520,7 +524,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Create angular variable in global registry and bind with front end Angular display system.
* Create angular variable in global scope and bind with front end Angular display system.
* If variable exists, it'll be overwritten.
* @param name name of the variable
* @param o value
@ -530,7 +534,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Create angular variable in local registry and bind with front end Angular display system.
* Create angular variable in local scope and bind with front end Angular display system.
* If variable exists, value will be overwritten and watcher will be added.
* @param name name of variable
* @param o value
@ -541,7 +545,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Create angular variable in global registry and bind with front end Angular display system.
* Create angular variable in global scope and bind with front end Angular display system.
* If variable exists, value will be overwritten and watcher will be added.
* @param name name of variable
* @param o value
@ -552,7 +556,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Add watcher into angular variable (local registry)
* Add watcher into angular variable (local scope)
* @param name name of the variable
* @param watcher watcher
*/
@ -561,7 +565,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Add watcher into angular variable (global registry)
* Add watcher into angular variable (global scope)
* @param name name of the variable
* @param watcher watcher
*/
@ -645,7 +649,7 @@ public class ZeppelinContext extends HashMap<String, Object> {
}
/**
* Create angular variable in local registry and bind with front end Angular display system.
* Create angular variable in notebook scope and bind with front end Angular display system.
* If variable exists, it'll be overwritten.
* @param name name of the variable
* @param o value
@ -653,15 +657,16 @@ public class ZeppelinContext extends HashMap<String, Object> {
private void angularBind(String name, Object o, String noteId) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
if (registry.get(name, noteId) == null) {
registry.add(name, o, noteId);
if (registry.get(name, noteId, null) == null) {
registry.add(name, o, noteId, null);
} else {
registry.get(name, noteId).set(o);
registry.get(name, noteId, null).set(o);
}
}
/**
* Create angular variable in local registry and bind with front end Angular display system.
* Create angular variable in notebook scope and bind with front end Angular display
* system.
* If variable exists, value will be overwritten and watcher will be added.
* @param name name of variable
* @param o value
@ -670,10 +675,10 @@ public class ZeppelinContext extends HashMap<String, Object> {
private void angularBind(String name, Object o, String noteId, AngularObjectWatcher watcher) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
if (registry.get(name, noteId) == null) {
registry.add(name, o, noteId);
if (registry.get(name, noteId, null) == null) {
registry.add(name, o, noteId, null);
} else {
registry.get(name, noteId).set(o);
registry.get(name, noteId, null).set(o);
}
angularWatch(name, watcher);
}
@ -686,8 +691,8 @@ public class ZeppelinContext extends HashMap<String, Object> {
private void angularWatch(String name, String noteId, AngularObjectWatcher watcher) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
if (registry.get(name, noteId) != null) {
registry.get(name, noteId).addWatcher(watcher);
if (registry.get(name, noteId, null) != null) {
registry.get(name, noteId, null).addWatcher(watcher);
}
}
@ -725,8 +730,8 @@ public class ZeppelinContext extends HashMap<String, Object> {
*/
private void angularUnwatch(String name, String noteId, AngularObjectWatcher watcher) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
if (registry.get(name, noteId) != null) {
registry.get(name, noteId).removeWatcher(watcher);
if (registry.get(name, noteId, null) != null) {
registry.get(name, noteId, null).removeWatcher(watcher);
}
}
@ -736,8 +741,8 @@ public class ZeppelinContext extends HashMap<String, Object> {
*/
private void angularUnwatch(String name, String noteId) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
if (registry.get(name, noteId) != null) {
registry.get(name, noteId).clearAllWatchers();
if (registry.get(name, noteId, null) != null) {
registry.get(name, noteId, null).clearAllWatchers();
}
}
@ -747,6 +752,6 @@ public class ZeppelinContext extends HashMap<String, Object> {
*/
private void angularUnbind(String name, String noteId) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
registry.remove(name, noteId);
registry.remove(name, noteId, null);
}
}

7
spark/src/main/resources/python/zeppelin_pyspark.py
View file

@ -36,10 +36,7 @@ class Logger(object):
self.out = ""
def write(self, message):
self.out = self.out + message
def get(self):
return self.out
intp.appendOutput(message)
def reset(self):
self.out = ""
@ -224,7 +221,7 @@ while True :
sc.setJobGroup(jobGroup, "Zeppelin")
eval(compiledCode)
intp.setStatementsFinished(output.get(), False)
intp.setStatementsFinished("", False)
except Py4JJavaError:
excInnerError = traceback.format_exc() # format_tb() does not return the inner exception
innerErrorStart = excInnerError.find("Py4JJavaError:")

2
spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java
View file

@ -60,7 +60,7 @@ public class DepInterpreterTest {
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), null);
}
@After

23
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -28,10 +28,7 @@ import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.After;
import org.junit.Before;
@ -79,9 +76,21 @@ public class SparkInterpreterTest {
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text",
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
@After

17
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -25,10 +25,7 @@ import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
@ -69,7 +66,17 @@ public class SparkSqlInterpreterTest {
}
context = new InterpreterContext("note", "id", "title", "text", new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
@After

164
tachyon/pom.xml Normal file
View file

@ -0,0 +1,164 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-tachyon</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Tachyon interpreter</name>
<url>http://www.apache.org</url>
<properties>
<tachyon.version>0.8.2</tachyon.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-shell</artifactId>
<version>${tachyon.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!-- TEST -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-servers</artifactId>
<version>${tachyon.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-minicluster</artifactId>
<version>${tachyon.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-underfs-local</artifactId>
<version>${tachyon.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/tachyon</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/tachyon</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

251
tachyon/src/main/java/org/apache/zeppelin/tachyon/TachyonInterpreter.java Normal file
View file

@ -0,0 +1,251 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.tachyon;
import java.io.IOException;
import java.io.PrintStream;
import java.io.ByteArrayOutputStream;
import java.util.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tachyon.conf.TachyonConf;
import tachyon.shell.TfsShell;
/**
* Tachyon interpreter for Zeppelin.
*/
public class TachyonInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(TachyonInterpreter.class);
protected static final String TACHYON_MASTER_HOSTNAME = "tachyon.master.hostname";
protected static final String TACHYON_MASTER_PORT = "tachyon.master.port";
private TfsShell tfs;
private int totalCommands = 0;
private int completedCommands = 0;
private final String tachyonMasterHostname;
private final String tachyonMasterPort;
protected final List<String> keywords = Arrays.asList("cat", "copyFromLocal",
"copyToLocal", "count", "du", "fileinfo", "free", "getUsedBytes",
"getCapacityBytes", "load", "loadMetadata", "location", "ls", "lsr",
"mkdir", "mount", "mv", "pin", "report", "request", "rm", "rmr",
"setTTL", "unsetTTL", "tail", "touch", "unmount", "unpin");
public TachyonInterpreter(Properties property) {
super(property);
tachyonMasterHostname = property.getProperty(TACHYON_MASTER_HOSTNAME);
tachyonMasterPort = property.getProperty(TACHYON_MASTER_PORT);
}
static {
Interpreter.register("tachyon", "tachyon",
TachyonInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(TACHYON_MASTER_HOSTNAME, "localhost", "Tachyon master hostname")
.add(TACHYON_MASTER_PORT, "19998", "Tachyon master port")
.build());
}
@Override
public void open() {
logger.info("Starting Tachyon shell to connect to " + tachyonMasterHostname +
" on port " + tachyonMasterPort);
System.setProperty(TACHYON_MASTER_HOSTNAME, tachyonMasterHostname);
System.setProperty(TACHYON_MASTER_PORT, tachyonMasterPort);
tfs = new TfsShell(new TachyonConf());
}
@Override
public void close() {
logger.info("Closing Tachyon shell");
try {
tfs.close();
} catch (IOException e) {
logger.error("Cannot close connection", e);
}
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
String[] lines = splitAndRemoveEmpty(st, "\n");
return interpret(lines, context);
}
private InterpreterResult interpret(String[] commands, InterpreterContext context) {
boolean isSuccess = true;
totalCommands = commands.length;
completedCommands = 0;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
PrintStream old = System.out;
System.setOut(ps);
for (String command : commands) {
int commandResuld = 1;
String[] args = splitAndRemoveEmpty(command, " ");
if (args.length > 0 && args[0].equals("help")) {
System.out.println(getCommandList());
} else {
commandResuld = tfs.run(args);
}
if (commandResuld != 0) {
isSuccess = false;
break;
} else {
completedCommands += 1;
}
System.out.println();
}
System.out.flush();
System.setOut(old);
if (isSuccess) {
return new InterpreterResult(Code.SUCCESS, baos.toString());
} else {
return new InterpreterResult(Code.ERROR, baos.toString());
}
}
private String[] splitAndRemoveEmpty(String st, String splitSeparator) {
String[] voices = st.split(splitSeparator);
ArrayList<String> result = new ArrayList<String>();
for (String voice : voices) {
if (!voice.trim().isEmpty()) {
result.add(voice);
}
}
return result.toArray(new String[result.size()]);
}
private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) {
ArrayList<String> result = new ArrayList<String>();
for (String st : sts) {
result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator)));
}
return result.toArray(new String[result.size()]);
}
@Override
public void cancel(InterpreterContext context) { }
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return completedCommands * 100 / totalCommands;
}
@Override
public List<String> completion(String buf, int cursor) {
String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
String lastWord = "";
if (words.length > 0) {
lastWord = words[ words.length - 1 ];
}
ArrayList<String> voices = new ArrayList<String>();
for (String command : keywords) {
if (command.startsWith(lastWord)) {
voices.add(command);
}
}
return voices;
}
private String getCommandList() {
StringBuilder sb = new StringBuilder();
sb.append("Commands list:");
sb.append("\n\t[help] - List all available commands.");
sb.append("\n\t[cat <path>] - Print the content of the file to the console.");
sb.append("\n\t[copyFromLocal <src> <remoteDst>] - Copy the specified file specified " +
"by \"source path\" to the path specified by \"remote path\". " +
"This command will fail if \"remote path\" already exists.");
sb.append("\n\t[copyToLocal <src> <localDst>] - Copy the specified file from the path " +
"specified by \"remote source\" to a local destination.");
sb.append("\n\t[count <path>] - Display the number of folders and files matching " +
"the specified prefix in \"path\".");
sb.append("\n\t[du <path>] - Display the size of a file or a directory specified " +
"by the input path.");
sb.append("\n\t[fileinfo <path>] - Print the information of the blocks of a specified file.");
sb.append("\n\t[free <file path|folder path>] - Free a file or all files under a " +
"directory from Tachyon. If the file/directory is also in under storage, " +
"it will still be available there.");
sb.append("\n\t[getUsedBytes] - Get number of bytes used in the TachyonFS.");
sb.append("\n\t[getCapacityBytes] - Get the capacity of the TachyonFS.");
sb.append("\n\t[load <path>] - Load the data of a file or a directory from under " +
"storage into Tachyon.");
sb.append("\n\t[loadMetadata <path>] - Load the metadata of a file or a directory " +
"from under storage into Tachyon.");
sb.append("\n\t[location <path>] - Display a list of hosts that have the file data.");
sb.append("\n\t[ls <path>] - List all the files and directories directly under the " +
"given path with information such as size.");
sb.append("\n\t[lsr <path>] - Recursively list all the files and directories under " +
"the given path with information such as size.");
sb.append("\n\t[mkdir <path>] - Create a directory under the given path, along with " +
"any necessary parent directories. This command will fail if the given " +
"path already exists.");
sb.append("\n\t[mount <tachyonPath> <ufsURI>] - Mount the underlying file system " +
"path \"uri\" into the Tachyon namespace as \"path\". The \"path\" is assumed " +
"not to exist and is created by the operation. No data or metadata is loaded " +
"from under storage into Tachyon. After a path is mounted, operations on objects " +
"under the mounted path are mirror to the mounted under storage.");
sb.append("\n\t[mv <src> <dst>] - Move a file or directory specified by \"source\" " +
"to a new location \"destination\". This command will fail if " +
"\"destination\" already exists.");
sb.append("\n\t[pin <path>] - Pin the given file to avoid evicting it from memory. " +
"If the given path is a directory, it recursively pins all the files contained " +
"and any new files created within this directory.");
sb.append("\n\t[report <path>] - Report to the master that a file is lost.");
sb.append("\n\t[request <tachyonaddress> <dependencyId>] - Request the file for " +
"a given dependency ID.");
sb.append("\n\t[rm <path>] - Remove a file. This command will fail if the given " +
"path is a directory rather than a file.");
sb.append("\n\t[rmr <path>] - Remove a file, or a directory with all the files and " +
"sub-directories that this directory contains.");
sb.append("\n\t[tail <path>] - Print the last 1KB of the specified file to the console.");
sb.append("\n\t[touch <path>] - Create a 0-byte file at the specified location.");
sb.append("\n\t[unmount <tachyonPath>] - Unmount the underlying file system path " +
"mounted in the Tachyon namespace as \"path\". Tachyon objects under \"path\" " +
"are removed from Tachyon, but they still exist in the previously " +
"mounted under storage.");
sb.append("\n\t[unpin <path>] - Unpin the given file to allow Tachyon to evict " +
"this file again. If the given path is a directory, it recursively unpins " +
"all files contained and any new files created within this directory.");
return sb.toString();
}
}

484
tachyon/src/test/java/org/apache/zeppelin/tachyon/TachyonInterpreterTest.java Normal file
View file

@ -0,0 +1,484 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.tachyon;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.*;
import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.client.TachyonFSTestUtils;
import tachyon.client.TachyonStorageType;
import tachyon.client.UnderStorageType;
import tachyon.client.file.FileInStream;
import tachyon.client.file.TachyonFile;
import tachyon.client.file.TachyonFileSystem;
import tachyon.client.file.options.InStreamOptions;
import tachyon.conf.TachyonConf;
import tachyon.exception.ExceptionMessage;
import tachyon.exception.TachyonException;
import tachyon.master.LocalTachyonCluster;
import tachyon.shell.TfsShell;
import tachyon.thrift.FileInfo;
import tachyon.util.FormatUtils;
import tachyon.util.io.BufferUtils;
import tachyon.util.io.PathUtils;
public class TachyonInterpreterTest {
private TachyonInterpreter tachyonInterpreter;
private static final int SIZE_BYTES = Constants.MB * 10;
private LocalTachyonCluster mLocalTachyonCluster = null;
private TachyonFileSystem mTfs = null;
@After
public final void after() throws Exception {
if (tachyonInterpreter != null) {
tachyonInterpreter.close();
}
mLocalTachyonCluster.stop();
}
@Before
public final void before() throws Exception {
mLocalTachyonCluster = new LocalTachyonCluster(SIZE_BYTES, 1000, Constants.GB);
mLocalTachyonCluster.start();
mTfs = mLocalTachyonCluster.getClient();
final Properties props = new Properties();
props.put(TachyonInterpreter.TACHYON_MASTER_HOSTNAME, mLocalTachyonCluster.getMasterHostname());
props.put(TachyonInterpreter.TACHYON_MASTER_PORT, mLocalTachyonCluster.getMasterPort() + "");
tachyonInterpreter = new TachyonInterpreter(props);
tachyonInterpreter.open();
}
@Test
public void testCompletion() {
List<String> expectedResultOne = Arrays.asList("cat", "copyFromLocal",
"copyToLocal", "count");
List<String> expectedResultTwo = Arrays.asList("copyFromLocal",
"copyToLocal", "count");
List<String> expectedResultThree = Arrays.asList("copyFromLocal", "copyToLocal");
List<String> expectedResultNone = new ArrayList<String>();
List<String> resultOne = tachyonInterpreter.completion("c", 0);
List<String> resultTwo = tachyonInterpreter.completion("co", 0);
List<String> resultThree = tachyonInterpreter.completion("copy", 0);
List<String> resultNotMatch = tachyonInterpreter.completion("notMatch", 0);
List<String> resultAll = tachyonInterpreter.completion("", 0);
Assert.assertEquals(expectedResultOne, resultOne);
Assert.assertEquals(expectedResultTwo, resultTwo);
Assert.assertEquals(expectedResultThree, resultThree);
Assert.assertEquals(expectedResultNone, resultNotMatch);
Assert.assertEquals(tachyonInterpreter.keywords, resultAll);
}
@Test
public void catDirectoryTest() throws IOException {
String expected = "Successfully created directory /testDir\n\n" +
"/testDir is not a file.\n";
InterpreterResult output = tachyonInterpreter.interpret("mkdir /testDir" +
"\ncat /testDir", null);
Assert.assertEquals(Code.ERROR, output.code());
Assert.assertEquals(expected, output.message());
}
@Test
public void catNotExistTest() throws IOException {
InterpreterResult output = tachyonInterpreter.interpret("cat /testFile", null);
Assert.assertEquals(Code.ERROR, output.code());
}
@Test
public void catTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 10);
InterpreterResult output = tachyonInterpreter.interpret("cat /testFile", null);
byte[] expected = BufferUtils.getIncreasingByteArray(10);
Assert.assertEquals(Code.SUCCESS, output.code());
Assert.assertArrayEquals(expected,
output.message().substring(0, output.message().length() - 1).getBytes());
}
@Test
public void copyFromLocalLargeTest() throws IOException, TachyonException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
fos.write(toWrite);
fos.close();
InterpreterResult output = tachyonInterpreter.interpret("copyFromLocal " +
testFile.getAbsolutePath() + " /testFile", null);
Assert.assertEquals(
"Copied " + testFile.getAbsolutePath() + " to /testFile\n\n",
output.message());
TachyonFile tFile = mTfs.open(new TachyonURI("/testFile"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(SIZE_BYTES, fileInfo.length);
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf()).setTachyonStorageType(
TachyonStorageType.NO_STORE).build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[SIZE_BYTES];
tfis.read(read);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
}
@Test
public void loadFileTest() throws IOException, TachyonException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", TachyonStorageType.NO_STORE,
UnderStorageType.SYNC_PERSIST, 10);
FileInfo fileInfo = mTfs.getInfo(file);
Assert.assertFalse(fileInfo.getInMemoryPercentage() == 100);
tachyonInterpreter.interpret("load /testFile", null);
fileInfo = mTfs.getInfo(file);
Assert.assertTrue(fileInfo.getInMemoryPercentage() == 100);
}
@Test
public void loadDirTest() throws IOException, TachyonException {
TachyonFile fileA = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA",
TachyonStorageType.NO_STORE, UnderStorageType.SYNC_PERSIST, 10);
TachyonFile fileB = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileB",
TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
FileInfo fileInfoA = mTfs.getInfo(fileA);
FileInfo fileInfoB = mTfs.getInfo(fileB);
Assert.assertFalse(fileInfoA.getInMemoryPercentage() == 100);
Assert.assertTrue(fileInfoB.getInMemoryPercentage() == 100);
tachyonInterpreter.interpret("load /testRoot", null);
fileInfoA = mTfs.getInfo(fileA);
fileInfoB = mTfs.getInfo(fileB);
Assert.assertTrue(fileInfoA.getInMemoryPercentage() == 100);
Assert.assertTrue(fileInfoB.getInMemoryPercentage() == 100);
}
@Test
public void copyFromLocalTest() throws IOException, TachyonException {
File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
testDir.mkdir();
File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
testDirInner.mkdir();
File testFile =
generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
generateFileContent("/testDir/testDirInner/testFile2",
BufferUtils.getIncreasingByteArray(10, 20));
InterpreterResult output = tachyonInterpreter.interpret("copyFromLocal " +
testFile.getParent() + " /testDir", null);
Assert.assertEquals(
"Copied " + testFile.getParent() + " to /testDir\n\n",
output.message());
TachyonFile file1 = mTfs.open(new TachyonURI("/testDir/testFile"));
TachyonFile file2 = mTfs.open(new TachyonURI("/testDir/testDirInner/testFile2"));
FileInfo fileInfo1 = mTfs.getInfo(file1);
FileInfo fileInfo2 = mTfs.getInfo(file2);
Assert.assertNotNull(fileInfo1);
Assert.assertNotNull(fileInfo2);
Assert.assertEquals(10, fileInfo1.length);
Assert.assertEquals(20, fileInfo2.length);
byte[] read = readContent(file1, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
read = readContent(file2, 20);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
}
@Test
public void copyFromLocalTestWithFullURI() throws IOException, TachyonException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String tachyonURI = "tachyon://" + mLocalTachyonCluster.getMasterHostname() + ":"
+ mLocalTachyonCluster.getMasterPort() + "/destFileURI";
InterpreterResult output = tachyonInterpreter.interpret("copyFromLocal " +
testFile.getPath() + " " + tachyonURI, null);
Assert.assertEquals(
"Copied " + testFile.getPath() + " to " + tachyonURI + "\n\n",
output.message());
TachyonFile file = mTfs.open(new TachyonURI("/destFileURI"));
FileInfo fileInfo = mTfs.getInfo(file);
Assert.assertEquals(10L, fileInfo.length);
byte[] read = readContent(file, 10);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
}
@Test
public void copyFromLocalFileToDstPathTest() throws IOException, TachyonException {
String dataString = "copyFromLocalFileToDstPathTest";
byte[] data = dataString.getBytes();
File localDir = new File(mLocalTachyonCluster.getTachyonHome() + "/localDir");
localDir.mkdir();
File localFile = generateFileContent("/localDir/testFile", data);
tachyonInterpreter.interpret("mkdir /dstDir", null);
tachyonInterpreter.interpret("copyFromLocal " + localFile.getPath() + " /dstDir", null);
TachyonFile file = mTfs.open(new TachyonURI("/dstDir/testFile"));
FileInfo fileInfo = mTfs.getInfo(file);
Assert.assertNotNull(fileInfo);
byte[] read = readContent(file, data.length);
Assert.assertEquals(new String(read), dataString);
}
@Test
public void copyToLocalLargeTest() throws IOException {
copyToLocalWithBytes(SIZE_BYTES);
}
@Test
public void copyToLocalTest() throws IOException {
copyToLocalWithBytes(10);
}
private void copyToLocalWithBytes(int bytes) throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, bytes);
InterpreterResult output = tachyonInterpreter.interpret("copyToLocal /testFile " +
mLocalTachyonCluster.getTachyonHome() + "/testFile", null);
Assert.assertEquals(
"Copied /testFile to " + mLocalTachyonCluster.getTachyonHome() + "/testFile\n\n",
output.message());
fileReadTest("/testFile", 10);
}
@Test
public void countNotExistTest() throws IOException {
InterpreterResult output = tachyonInterpreter.interpret("count /NotExistFile", null);
Assert.assertEquals(Code.ERROR, output.code());
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message());
}
@Test
public void countTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 10);
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testDir/testFileB", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 20);
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileB", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 30);
InterpreterResult output = tachyonInterpreter.interpret("count /testRoot", null);
String expected = "";
String format = "%-25s%-25s%-15s\n";
expected += String.format(format, "File Count", "Folder Count", "Total Bytes");
expected += String.format(format, 3, 2, 60);
expected += "\n";
Assert.assertEquals(expected, output.message());
}
@Test
public void fileinfoNotExistTest() throws IOException {
InterpreterResult output = tachyonInterpreter.interpret("fileinfo /NotExistFile", null);
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message());
Assert.assertEquals(Code.ERROR, output.code());
}
@Test
public void locationNotExistTest() throws IOException {
InterpreterResult output = tachyonInterpreter.interpret("location /NotExistFile", null);
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message());
Assert.assertEquals(Code.ERROR, output.code());
}
@Test
public void lsTest() throws IOException, TachyonException {
FileInfo[] files = new FileInfo[3];
TachyonFile fileA = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA",
TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
files[0] = mTfs.getInfo(fileA);
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testDir/testFileB", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 20);
files[1] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir")));
TachyonFile fileC = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileC",
TachyonStorageType.NO_STORE, UnderStorageType.SYNC_PERSIST, 30);
files[2] = mTfs.getInfo(fileC);
InterpreterResult output = tachyonInterpreter.interpret("ls /testRoot", null);
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected += String.format(format, FormatUtils.getSizeFromBytes(10),
TfsShell.convertMsToDate(files[0].getCreationTimeMs()), "In Memory", "/testRoot/testFileA");
expected += String.format(format, FormatUtils.getSizeFromBytes(0),
TfsShell.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
expected += String.format(format, FormatUtils.getSizeFromBytes(30),
TfsShell.convertMsToDate(files[2].getCreationTimeMs()), "Not In Memory",
"/testRoot/testFileC");
expected += "\n";
Assert.assertEquals(Code.SUCCESS, output.code());
Assert.assertEquals(expected, output.message());
}
@Test
public void lsrTest() throws IOException, TachyonException {
FileInfo[] files = new FileInfo[4];
TachyonFile fileA = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA",
TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
files[0] = mTfs.getInfo(fileA);
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testDir/testFileB", TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST, 20);
files[1] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir")));
files[2] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir/testFileB")));
TachyonFile fileC = TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileC",
TachyonStorageType.NO_STORE, UnderStorageType.SYNC_PERSIST, 30);
files[3] = mTfs.getInfo(fileC);
InterpreterResult output = tachyonInterpreter.interpret("lsr /testRoot", null);
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected +=
String.format(format, FormatUtils.getSizeFromBytes(10),
TfsShell.convertMsToDate(files[0].getCreationTimeMs()), "In Memory",
"/testRoot/testFileA");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(0),
TfsShell.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(20),
TfsShell.convertMsToDate(files[2].getCreationTimeMs()), "In Memory",
"/testRoot/testDir/testFileB");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(30),
TfsShell.convertMsToDate(files[3].getCreationTimeMs()), "Not In Memory",
"/testRoot/testFileC");
expected += "\n";
Assert.assertEquals(expected, output.message());
}
@Test
public void mkdirComplexPathTest() throws IOException, TachyonException {
InterpreterResult output = tachyonInterpreter.interpret(
"mkdir /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File", null);
TachyonFile tFile = mTfs.open(new TachyonURI("/Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(
"Successfully created directory /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File\n\n",
output.message());
Assert.assertTrue(fileInfo.isIsFolder());
}
@Test
public void mkdirExistingTest() throws IOException {
String command = "mkdir /festFile1";
Assert.assertEquals(Code.SUCCESS, tachyonInterpreter.interpret(command, null).code());
Assert.assertEquals(Code.SUCCESS, tachyonInterpreter.interpret(command, null).code());
}
@Test
public void mkdirInvalidPathTest() throws IOException {
Assert.assertEquals(
Code.ERROR,
tachyonInterpreter.interpret("mkdir /test File Invalid Path", null).code());
}
@Test
public void mkdirShortPathTest() throws IOException, TachyonException {
InterpreterResult output = tachyonInterpreter.interpret("mkdir /root/testFile1", null);
TachyonFile tFile = mTfs.open(new TachyonURI("/root/testFile1"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(
"Successfully created directory /root/testFile1\n\n",
output.message());
Assert.assertTrue(fileInfo.isIsFolder());
}
@Test
public void mkdirTest() throws IOException, TachyonException {
String qualifiedPath =
"tachyon://" + mLocalTachyonCluster.getMasterHostname() + ":"
+ mLocalTachyonCluster.getMasterPort() + "/root/testFile1";
InterpreterResult output = tachyonInterpreter.interpret("mkdir " + qualifiedPath, null);
TachyonFile tFile = mTfs.open(new TachyonURI("/root/testFile1"));
FileInfo fileInfo = mTfs.getInfo(tFile);
Assert.assertNotNull(fileInfo);
Assert.assertEquals(
"Successfully created directory " + qualifiedPath + "\n\n",
output.message());
Assert.assertTrue(fileInfo.isIsFolder());
}
private File generateFileContent(String path, byte[] toWrite)
throws IOException, FileNotFoundException {
File testFile = new File(mLocalTachyonCluster.getTachyonHome() + path);
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);
fos.write(toWrite);
fos.close();
return testFile;
}
private byte[] readContent(TachyonFile tFile, int length) throws IOException, TachyonException {
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf()).setTachyonStorageType(
TachyonStorageType.NO_STORE).build();
FileInStream tfis = mTfs.getInStream(tFile, options);
byte[] read = new byte[length];
tfis.read(read);
return read;
}
private void fileReadTest(String fileName, int size) throws IOException {
File testFile = new File(PathUtils.concatPath(mLocalTachyonCluster.getTachyonHome(), fileName));
FileInputStream fis = new FileInputStream(testFile);
byte[] read = new byte[size];
fis.read(read);
fis.close();
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(size, read));
}
}

4
testing/startSparkCluster.sh
View file

@ -34,7 +34,7 @@ if [ ! -d "${SPARK_HOME}" ]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [ $? -eq 0 ]; then
# spark 1.1.x and spark 1.2.x can be downloaded from archive
wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
else
# spark 1.3.x and later can be downloaded from mirror
# get download address from mirror
@ -42,7 +42,7 @@ if [ ! -d "${SPARK_HOME}" ]; then
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
wget "${PREFFERED}${PATHINFO}"
wget -q "${PREFFERED}${PATHINFO}"
fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi

4
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -96,6 +96,10 @@ The following components are provided under Apache License.
(Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org)
(Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org)
(Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.4.1 - https://github.com/google/protobuf/releases)
(Apache 2.0) Tachyon Shell (org.tachyonproject:tachyon-shell:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Servers (org.tachyonproject:tachyon-servers:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Minicluster (org.tachyonproject:tachyon-minicluster:0.8.2 - http://tachyon-project.org)
(Apache 2.0) Tachyon Underfs Local (org.tachyonproject:tachyon-underfs-local:0.8.2 - http://tachyon-project.org)

101
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java
View file

@ -26,7 +26,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* AngularObject provides binding between back-end (interpreter) and front-end
* User provided object will automatically synchronized with front-end side.
* i.e. update from back-end will be sent to front-end, update from front-end will sent-to backend
*
* @param <T>
*/
@ -39,27 +41,70 @@ public class AngularObject<T> {
= new LinkedList<AngularObjectWatcher>();
private String noteId; // noteId belonging to. null for global scope
private String paragraphId; // paragraphId belongs to. null for notebook scope
protected AngularObject(String name, T o, String noteId,
/**
* To create new AngularObject, use AngularObjectRegistry.add()
*
* @param name name of object
* @param o reference to user provided object to sent to front-end
* @param noteId noteId belongs to. can be null
* @param paragraphId paragraphId belongs to. can be null
* @param listener event listener
*/
protected AngularObject(String name, T o, String noteId, String paragraphId,
AngularObjectListener listener) {
this.name = name;
this.noteId = noteId;
this.paragraphId = paragraphId;
this.listener = listener;
object = o;
}
/**
* Get name of this object
* @return name
*/
public String getName() {
return name;
}
/**
* Set noteId
* @param noteId noteId belongs to. can be null
*/
public void setNoteId(String noteId) {
this.noteId = noteId;
}
/**
* Get noteId
* @return noteId
*/
public String getNoteId() {
return noteId;
}
/**
* get ParagraphId
* @return paragraphId
*/
public String getParagraphId() {
return paragraphId;
}
/**
* Set paragraphId
* @param paragraphId paragraphId. can be null
*/
public void setParagraphId(String paragraphId) {
this.paragraphId = paragraphId;
}
/**
* Check if it is global scope object
* @return true it is global scope
*/
public boolean isGlobal() {
return noteId == null;
}
@ -70,26 +115,47 @@ public class AngularObject<T> {
AngularObject ao = (AngularObject) o;
if (noteId == null && ao.noteId == null ||
(noteId != null && ao.noteId != null && noteId.equals(ao.noteId))) {
return name.equals(ao.name);
if (paragraphId == null && ao.paragraphId == null ||
(paragraphId != null && ao.paragraphId != null && paragraphId.equals(ao.paragraphId))) {
return name.equals(ao.name);
}
}
}
return false;
}
/**
* Get value
* @return
*/
public Object get() {
return object;
}
/**
* fire updated() event for listener
* Note that it does not invoke watcher.watch()
*/
public void emit(){
if (listener != null) {
listener.updated(this);
}
}
/**
* Set value
* @param o reference to new user provided object
*/
public void set(T o) {
set(o, true);
}
/**
* Set value
* @param o reference to new user provided object
* @param emit false on skip firing event for listener. note that it does not skip invoke
* watcher.watch() in any case
*/
public void set(T o, boolean emit) {
final T before = object;
final T after = o;
@ -119,26 +185,47 @@ public class AngularObject<T> {
}
}
/**
* Set event listener for this object
* @param listener
*/
public void setListener(AngularObjectListener listener) {
this.listener = listener;
}
/**
* Get event listener of this object
* @return event listener
*/
public AngularObjectListener getListener() {
return listener;
}
/**
* Add a watcher for this object.
* Multiple watcher can be registered.
*
* @param watcher watcher to add
*/
public void addWatcher(AngularObjectWatcher watcher) {
synchronized (watchers) {
watchers.add(watcher);
}
}
/**
* Remove a watcher from this object
* @param watcher watcher to remove
*/
public void removeWatcher(AngularObjectWatcher watcher) {
synchronized (watchers) {
watchers.remove(watcher);
}
}
/**
* Remove all watchers from this object
*/
public void clearAllWatchers() {
synchronized (watchers) {
watchers.clear();

143
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java
View file

@ -26,9 +26,10 @@ import java.util.Map;
/**
* AngularObjectRegistry keeps all the object that binded to Angular Display System.
* AngularObjectRegistry is created per interpreter group.
* It keeps two different set of AngularObjects :
* - globalRegistry: Shared to all notebook that uses the same interpreter group
* - localRegistry: AngularObject is valid only inside of a single notebook
* It provides three different scope of AngularObjects :
* - Paragraphscope : AngularObject is valid in specific paragraph
* - Notebook scope: AngularObject is valid in a single notebook
* - Global scope : Shared to all notebook that uses the same interpreter group
*/
public class AngularObjectRegistry {
Map<String, Map<String, AngularObject>> registry =
@ -60,26 +61,36 @@ public class AngularObjectRegistry {
/**
* Add object into registry
* @param name
* @param o
* @param noteId noteId belonging to. null for global object.
* @return
*
* Paragraph scope when noteId and paragraphId both not null
* Notebook scope when paragraphId is null
* Global scope when noteId and paragraphId both null
*
* @param name Name of object
* @param o Reference to the object
* @param noteId noteId belonging to. null for global scope
* @param paragraphId paragraphId belongs to. null for notebook scope
* @return AngularObject that added
*/
public AngularObject add(String name, Object o, String noteId) {
return add(name, o, noteId, true);
public AngularObject add(String name, Object o, String noteId, String paragraphId) {
return add(name, o, noteId, paragraphId, true);
}
private String getRegistryKey(String noteId) {
private String getRegistryKey(String noteId, String paragraphId) {
if (noteId == null) {
return GLOBAL_KEY;
} else {
return noteId;
if (paragraphId == null) {
return noteId;
} else {
return noteId + "_" + paragraphId;
}
}
}
private Map<String, AngularObject> getRegistryForKey(String noteId) {
private Map<String, AngularObject> getRegistryForKey(String noteId, String paragraphId) {
synchronized (registry) {
String key = getRegistryKey(noteId);
String key = getRegistryKey(noteId, paragraphId);
if (!registry.containsKey(key)) {
registry.put(key, new HashMap<String, AngularObject>());
}
@ -87,12 +98,27 @@ public class AngularObjectRegistry {
return registry.get(key);
}
}
public AngularObject add(String name, Object o, String noteId, boolean emit) {
AngularObject ao = createNewAngularObject(name, o, noteId);
/**
* Add object into registry
*
* Paragraph scope when noteId and paragraphId both not null
* Notebook scope when paragraphId is null
* Global scope when noteId and paragraphId both null
*
* @param name Name of object
* @param o Reference to the object
* @param noteId noteId belonging to. null for global scope
* @param paragraphId paragraphId belongs to. null for notebook scope
* @param emit skip firing onAdd event on false
* @return AngularObject that added
*/
public AngularObject add(String name, Object o, String noteId, String paragraphId,
boolean emit) {
AngularObject ao = createNewAngularObject(name, o, noteId, paragraphId);
synchronized (registry) {
Map<String, AngularObject> noteLocalRegistry = getRegistryForKey(noteId);
Map<String, AngularObject> noteLocalRegistry = getRegistryForKey(noteId, paragraphId);
noteLocalRegistry.put(name, ao);
if (listener != null && emit) {
listener.onAdd(interpreterId, ao);
@ -102,49 +128,90 @@ public class AngularObjectRegistry {
return ao;
}
protected AngularObject createNewAngularObject(String name, Object o, String noteId) {
return new AngularObject(name, o, noteId, angularObjectListener);
protected AngularObject createNewAngularObject(String name, Object o, String noteId,
String paragraphId) {
return new AngularObject(name, o, noteId, paragraphId, angularObjectListener);
}
protected AngularObjectListener getAngularObjectListener() {
return angularObjectListener;
}
public AngularObject remove(String name, String noteId) {
return remove(name, noteId, true);
/**
* Remove a object from registry
*
* @param name Name of object to remove
* @param noteId noteId belongs to. null for global scope
* @param paragraphId paragraphId belongs to. null for notebook scope
* @return removed object. null if object is not found in registry
*/
public AngularObject remove(String name, String noteId, String paragraphId) {
return remove(name, noteId, paragraphId, true);
}
public AngularObject remove(String name, String noteId, boolean emit) {
/**
* Remove a object from registry
*
* @param name Name of object to remove
* @param noteId noteId belongs to. null for global scope
* @param paragraphId paragraphId belongs to. null for notebook scope
* @param emit skip fireing onRemove event on false
* @return removed object. null if object is not found in registry
*/
public AngularObject remove(String name, String noteId, String paragraphId, boolean emit) {
synchronized (registry) {
Map<String, AngularObject> r = getRegistryForKey(noteId);
Map<String, AngularObject> r = getRegistryForKey(noteId, paragraphId);
AngularObject o = r.remove(name);
if (listener != null && emit) {
listener.onRemove(interpreterId, name, noteId);;
listener.onRemove(interpreterId, name, noteId, paragraphId);;
}
return o;
}
}
public void removeAll(String noteId) {
/**
* Remove all angular object in the scope.
*
* Remove all paragraph scope angular object when noteId and paragraphId both not null
* Remove all notebook scope angular object when paragraphId is null
* Remove all global scope angular objects when noteId and paragraphId both null
*
* @param noteId noteId
* @param paragraphId paragraphId
*/
public void removeAll(String noteId, String paragraphId) {
synchronized (registry) {
List<AngularObject> all = getAll(noteId);
List<AngularObject> all = getAll(noteId, paragraphId);
for (AngularObject ao : all) {
remove(ao.getName(), noteId);
remove(ao.getName(), noteId, paragraphId);
}
}
}
public AngularObject get(String name, String noteId) {
/**
* Get a object from registry
* @param name name of object
* @param noteId noteId that belongs to
* @param paragraphId paragraphId that belongs to
* @return angularobject. null when not found
*/
public AngularObject get(String name, String noteId, String paragraphId) {
synchronized (registry) {
Map<String, AngularObject> r = getRegistryForKey(noteId);
Map<String, AngularObject> r = getRegistryForKey(noteId, paragraphId);
return r.get(name);
}
}
public List<AngularObject> getAll(String noteId) {
/**
* Get all object in the scope
* @param noteId noteId that belongs to
* @param paragraphId paragraphId that belongs to
* @return all angularobject in the scope
*/
public List<AngularObject> getAll(String noteId, String paragraphId) {
List<AngularObject> all = new LinkedList<AngularObject>();
synchronized (registry) {
Map<String, AngularObject> r = getRegistryForKey(noteId);
Map<String, AngularObject> r = getRegistryForKey(noteId, paragraphId);
if (r != null) {
all.addAll(r.values());
}
@ -153,20 +220,24 @@ public class AngularObjectRegistry {
}
/**
* Get all object with global merged
* Get all angular object related to specific note.
* That includes all global scope objects, notebook scope objects and paragraph scope objects
* belongs to the noteId.
*
* @param noteId
* @return
*/
public List<AngularObject> getAllWithGlobal(String noteId) {
List<AngularObject> all = new LinkedList<AngularObject>();
synchronized (registry) {
Map<String, AngularObject> global = getRegistryForKey(null);
Map<String, AngularObject> global = getRegistryForKey(null, null);
if (global != null) {
all.addAll(global.values());
}
Map<String, AngularObject> local = getRegistryForKey(noteId);
if (local != null) {
all.addAll(local.values());
for (String key : registry.keySet()) {
if (key.startsWith(noteId)) {
all.addAll(registry.get(key).values());
}
}
}
return all;

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistryListener.java
View file

@ -24,5 +24,5 @@ package org.apache.zeppelin.display;
public interface AngularObjectRegistryListener {
public void onAdd(String interpreterGroupId, AngularObject object);
public void onUpdate(String interpreterGroupId, AngularObject object);
public void onRemove(String interpreterGroupId, String name, String noteId);
public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId);
}

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.display.GUI;
public class InterpreterContext {
private static final ThreadLocal<InterpreterContext> threadIC =
new ThreadLocal<InterpreterContext>();
public final InterpreterOutput out;
public static InterpreterContext get() {
return threadIC.get();
@ -58,7 +59,8 @@ public class InterpreterContext {
Map<String, Object> config,
GUI gui,
AngularObjectRegistry angularObjectRegistry,
List<InterpreterContextRunner> runners
List<InterpreterContextRunner> runners,
InterpreterOutput out
) {
this.noteId = noteId;
this.paragraphId = paragraphId;
@ -68,6 +70,7 @@ public class InterpreterContext {
this.gui = gui;
this.angularObjectRegistry = angularObjectRegistry;
this.runners = runners;
this.out = out;
}

249
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java Normal file
View file

@ -0,0 +1,249 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
/**
* InterpreterOutput is OutputStream that supposed to print content on notebook
* in addition to InterpreterResult which used to return from Interpreter.interpret().
*/
public class InterpreterOutput extends OutputStream {
Logger logger = LoggerFactory.getLogger(InterpreterOutput.class);
private final int NEW_LINE_CHAR = '\n';
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
private final List<Object> outList = new LinkedList<Object>();
private InterpreterOutputChangeWatcher watcher;
private final InterpreterOutputListener flushListener;
private InterpreterResult.Type type = InterpreterResult.Type.TEXT;
private boolean firstWrite = true;
public InterpreterOutput(InterpreterOutputListener flushListener) {
this.flushListener = flushListener;
clear();
}
public InterpreterOutput(InterpreterOutputListener flushListener,
InterpreterOutputChangeListener listener) throws IOException {
this.flushListener = flushListener;
clear();
watcher = new InterpreterOutputChangeWatcher(listener);
watcher.start();
}
public InterpreterResult.Type getType() {
return type;
}
public void setType(InterpreterResult.Type type) {
if (this.type != type) {
clear();
flushListener.onUpdate(this, new byte[]{});
this.type = type;
}
}
public void clear() {
synchronized (outList) {
type = InterpreterResult.Type.TEXT;
buffer.reset();
outList.clear();
if (watcher != null) {
watcher.clear();
}
}
}
@Override
public void write(int b) throws IOException {
synchronized (outList) {
buffer.write(b);
if (b == NEW_LINE_CHAR) {
// first time use of this outputstream.
if (firstWrite) {
// clear the output on gui
flushListener.onUpdate(this, new byte[]{});
firstWrite = false;
}
flush();
}
}
}
private byte [] detectTypeFromLine(byte [] byteArray) {
// check output type directive
String line = new String(byteArray);
for (InterpreterResult.Type t : InterpreterResult.Type.values()) {
String typeString = '%' + t.name().toLowerCase();
if ((typeString + "\n").equals(line)) {
setType(t);
byteArray = null;
break;
} else if (line.startsWith(typeString + " ")) {
setType(t);
byteArray = line.substring(typeString.length() + 1).getBytes();
break;
}
}
return byteArray;
}
@Override
public void write(byte [] b) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte [] b, int off, int len) throws IOException {
synchronized (outList) {
for (int i = off; i < len; i++) {
write(b[i]);
}
}
}
/**
* In dev mode, it monitors file and update ZeppelinServer
* @param file
* @throws IOException
*/
public void write(File file) throws IOException {
outList.add(file);
if (watcher != null) {
watcher.watch(file);
}
}
public void write(String string) throws IOException {
write(string.getBytes());
}
/**
* write contents in the resource file in the classpath
* @param url
* @throws IOException
*/
public void write(URL url) throws IOException {
if ("file".equals(url.getProtocol())) {
write(new File(url.getPath()));
} else {
outList.add(url);
}
}
public void writeResource(String resourceName) throws IOException {
// search file under resource dir first for dev mode
File mainResource = new File("./src/main/resources/" + resourceName);
File testResource = new File("./src/test/resources/" + resourceName);
if (mainResource.isFile()) {
write(mainResource);
} else if (testResource.isFile()) {
write(testResource);
} else {
// search from classpath
ClassLoader cl = Thread.currentThread().getContextClassLoader();
if (cl == null) {
cl = this.getClass().getClassLoader();
}
if (cl == null) {
cl = ClassLoader.getSystemClassLoader();
}
write(cl.getResource(resourceName));
}
}
public byte[] toByteArray() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
List<Object> all = new LinkedList<Object>();
synchronized (outList) {
all.addAll(outList);
}
for (Object o : all) {
if (o instanceof File) {
File f = (File) o;
FileInputStream fin = new FileInputStream(f);
copyStream(fin, out);
fin.close();
} else if (o instanceof byte[]) {
out.write((byte[]) o);
} else if (o instanceof Integer) {
out.write((int) o);
} else if (o instanceof URL) {
InputStream fin = ((URL) o).openStream();
copyStream(fin, out);
fin.close();
} else {
// can not handle the object
}
}
out.close();
return out.toByteArray();
}
public void flush() throws IOException {
synchronized (outList) {
buffer.flush();
byte[] bytes = buffer.toByteArray();
bytes = detectTypeFromLine(bytes);
if (bytes != null) {
outList.add(bytes);
if (type == InterpreterResult.Type.TEXT) {
flushListener.onAppend(this, bytes);
}
}
buffer.reset();
}
}
private void copyStream(InputStream in, OutputStream out) throws IOException {
int bufferSize = 8192;
byte[] buffer = new byte[bufferSize];
while (true) {
int bytesRead = in.read(buffer);
if (bytesRead == -1) {
break;
} else {
out.write(buffer, 0, bytesRead);
}
}
}
@Override
public void close() throws IOException {
flush();
if (watcher != null) {
watcher.clear();
watcher.shutdown();
}
}
}

27
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeListener.java Normal file
View file

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import java.io.File;
/**
* InterpreterOutputChangeListener
*/
public interface InterpreterOutputChangeListener {
public void fileChanged(File file);
}

140
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcher.java Normal file
View file

@ -0,0 +1,140 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import java.io.File;
import java.io.IOException;
import java.nio.file.ClosedWatchServiceException;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Watch the change for the development mode support
*/
public class InterpreterOutputChangeWatcher extends Thread {
Logger logger = LoggerFactory.getLogger(InterpreterOutputChangeWatcher.class);
private WatchService watcher;
private final List<File> watchFiles = new LinkedList<File>();
private final Map<WatchKey, File> watchKeys = new HashMap<WatchKey, File>();
private InterpreterOutputChangeListener listener;
private boolean stop;
public InterpreterOutputChangeWatcher(InterpreterOutputChangeListener listener)
throws IOException {
watcher = FileSystems.getDefault().newWatchService();
this.listener = listener;
}
public void watch(File file) throws IOException {
String dirString;
if (file.isFile()) {
dirString = file.getParentFile().getAbsolutePath();
} else {
throw new IOException(file.getName() + " is not a file");
}
if (dirString == null) {
dirString = "/";
}
Path dir = FileSystems.getDefault().getPath(dirString);
logger.info("watch " + dir);
WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
synchronized (watchKeys) {
watchKeys.put(key, new File(dirString));
watchFiles.add(file);
}
}
public void clear() {
synchronized (watchKeys) {
for (WatchKey key : watchKeys.keySet()) {
key.cancel();
}
watchKeys.clear();
watchFiles.clear();
}
}
public void shutdown() throws IOException {
stop = true;
clear();
watcher.close();
}
public void run() {
while (!stop) {
WatchKey key = null;
try {
key = watcher.poll(1, TimeUnit.SECONDS);
} catch (InterruptedException | ClosedWatchServiceException e) {
break;
}
if (key == null) {
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind<?> kind = event.kind();
if (kind == OVERFLOW) {
continue;
}
WatchEvent<Path> ev = (WatchEvent<Path>) event;
Path filename = ev.context();
// search for filename
synchronized (watchKeys) {
for (File f : watchFiles) {
if (f.getName().compareTo(filename.toString()) == 0) {
File changedFile;
if (filename.isAbsolute()) {
changedFile = new File(filename.toString());
} else {
changedFile = new File(watchKeys.get(key), filename.toString());
}
logger.info("File change detected " + changedFile.getAbsolutePath());
if (listener != null) {
listener.fileChanged(changedFile);
}
}
}
}
}
boolean valid = key.reset();
if (!valid) {
break;
}
}
}
}

34
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputListener.java Normal file
View file

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
/**
* Listen InterpreterOutput buffer flush
*/
public interface InterpreterOutputListener {
/**
* called when newline is detected
* @param line
*/
public void onAppend(InterpreterOutput out, byte[] line);
/**
* when entire output is updated. eg) after detecting new display system
* @param output
*/
public void onUpdate(InterpreterOutput out, byte[] output);
}

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java
View file

@ -146,4 +146,8 @@ public class InterpreterResult implements Serializable {
this.type = type;
return this;
}
public String toString() {
return "%" + type.name().toLowerCase() + " " + msg;
}
}

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
View file

@ -21,16 +21,17 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectListener;
/**
*
* Proxy for AngularObject that exists in remote interpreter process
*/
public class RemoteAngularObject extends AngularObject {
private transient RemoteInterpreterProcess remoteInterpreterProcess;
RemoteAngularObject(String name, Object o, String noteId, String interpreterGroupId,
RemoteAngularObject(String name, Object o, String noteId, String paragraphId, String
interpreterGroupId,
AngularObjectListener listener,
RemoteInterpreterProcess remoteInterpreterProcess) {
super(name, o, noteId, listener);
super(name, o, noteId, paragraphId, listener);
this.remoteInterpreterProcess = remoteInterpreterProcess;
}
@ -44,7 +45,8 @@ public class RemoteAngularObject extends AngularObject {
if (emitRemoteProcess) {
// send updated value to remote interpreter
remoteInterpreterProcess.updateRemoteAngularObject(getName(), getNoteId(), o);
remoteInterpreterProcess.updateRemoteAngularObject(getName(), getNoteId(), getParagraphId()
, o);
}
}
}

28
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
View file

@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
/**
*
* Proxy for AngularObjectRegistry that exists in remote interpreter process
*/
public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
Logger logger = LoggerFactory.getLogger(RemoteAngularObjectRegistry.class);
@ -70,7 +70,8 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param noteId
* @return
*/
public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId) {
public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId, String
paragraphId) {
Gson gson = new Gson();
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (!remoteInterpreterProcess.isRunning()) {
@ -81,8 +82,8 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectAdd(name, noteId, gson.toJson(o));
return super.add(name, o, noteId, true);
client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
return super.add(name, o, noteId, paragraphId, true);
} catch (TException e) {
broken = true;
logger.error("Error", e);
@ -101,9 +102,11 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* this method should be used instead of remove()
* @param name
* @param noteId
* @param paragraphId
* @return
*/
public AngularObject removeAndNotifyRemoteProcess(String name, String noteId) {
public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String
paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (!remoteInterpreterProcess.isRunning()) {
return null;
@ -113,8 +116,8 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectRemove(name, noteId);
return super.remove(name, noteId);
client.angularObjectRemove(name, noteId, paragraphId);
return super.remove(name, noteId, paragraphId);
} catch (TException e) {
broken = true;
logger.error("Error", e);
@ -128,20 +131,21 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
return null;
}
public void removeAllAndNotifyRemoteProcess(String noteId) {
List<AngularObject> all = getAll(noteId);
public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {
List<AngularObject> all = getAll(noteId, paragraphId);
for (AngularObject ao : all) {
removeAndNotifyRemoteProcess(ao.getName(), noteId);
removeAndNotifyRemoteProcess(ao.getName(), noteId, paragraphId);
}
}
@Override
protected AngularObject createNewAngularObject(String name, Object o, String noteId) {
protected AngularObject createNewAngularObject(String name, Object o, String noteId, String
paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
throw new RuntimeException("Remote Interpreter process not found");
}
return new RemoteAngularObject(name, o, noteId, getInterpreterGroupId(),
return new RemoteAngularObject(name, o, noteId, paragraphId, getInterpreterGroupId(),
getAngularObjectListener(),
getRemoteInterpreterProcess());
}

36
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -48,6 +48,7 @@ import com.google.gson.reflect.TypeToken;
*
*/
public class RemoteInterpreter extends Interpreter {
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
Gson gson = new Gson();
private String interpreterRunner;
@ -56,36 +57,42 @@ public class RemoteInterpreter extends Interpreter {
FormType formType;
boolean initialized;
private Map<String, String> env;
private int connectTimeout;
private int maxPoolSize;
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
int connectTimeout) {
String className,
String interpreterRunner,
String interpreterPath,
int connectTimeout,
int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
env = new HashMap<String, String>();
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
Map<String, String> env,
int connectTimeout) {
String className,
String interpreterRunner,
String interpreterPath,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener) {
super(property);
this.className = className;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.env = env;
this.connectTimeout = connectTimeout;
this.maxPoolSize = 10;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
@Override
@ -103,7 +110,8 @@ public class RemoteInterpreter extends Interpreter {
if (intpGroup.getRemoteInterpreterProcess() == null) {
// create new remote process
RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess(
interpreterRunner, interpreterPath, env, connectTimeout);
interpreterRunner, interpreterPath, env, connectTimeout,
remoteInterpreterProcessListener);
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}
@ -119,7 +127,7 @@ public class RemoteInterpreter extends Interpreter {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
int rc = interpreterProcess.reference(getInterpreterGroup());
interpreterProcess.setMaxPoolSize(this.maxPoolSize);
synchronized (interpreterProcess) {
// when first process created
if (rc == 1) {
@ -325,7 +333,7 @@ public class RemoteInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
int maxConcurrency = 10;
int maxConcurrency = maxPoolSize;
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null) {
return null;

33
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -18,29 +18,35 @@
package org.apache.zeppelin.interpreter.remote;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.thrift.TException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/**
*
*/
public class RemoteInterpreterEventPoller extends Thread {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class);
private final RemoteInterpreterProcessListener listener;
private volatile boolean shutdown;
private RemoteInterpreterProcess interpreterProcess;
private InterpreterGroup interpreterGroup;
public RemoteInterpreterEventPoller() {
public RemoteInterpreterEventPoller(RemoteInterpreterProcessListener listener) {
this.listener = listener;
shutdown = false;
}
@ -88,12 +94,12 @@ public class RemoteInterpreterEventPoller extends Thread {
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) {
AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
angularObjectRegistry.add(angularObject.getName(),
angularObject.get(), angularObject.getNoteId());
angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) {
AngularObject angularObject = gson.fromJson(event.getData(),
AngularObject.class);
AngularObject localAngularObject = angularObjectRegistry.get(
angularObject.getName(), angularObject.getNoteId());
angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId());
if (localAngularObject instanceof RemoteAngularObject) {
// to avoid ping-pong loop
((RemoteAngularObject) localAngularObject).set(
@ -103,13 +109,32 @@ public class RemoteInterpreterEventPoller extends Thread {
}
} else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) {
AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class);
angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId());
angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(),
angularObject.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) {
InterpreterContextRunner runnerFromRemote = gson.fromJson(
event.getData(), RemoteInterpreterContextRunner.class);
interpreterProcess.getInterpreterContextRunnerPool().run(
runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId());
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) {
// on output append
Map<String, String> outputAppend = gson.fromJson(
event.getData(), new TypeToken<Map<String, String>>() {}.getType());
String noteId = outputAppend.get("noteId");
String paragraphId = outputAppend.get("paragraphId");
String outputToAppend = outputAppend.get("data");
listener.onOutputAppend(noteId, paragraphId, outputToAppend);
} else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) {
// on output update
Map<String, String> outputAppend = gson.fromJson(
event.getData(), new TypeToken<Map<String, String>>() {}.getType());
String noteId = outputAppend.get("noteId");
String paragraphId = outputAppend.get("paragraphId");
String outputToUpdate = outputAppend.get("data");
listener.onOutputUpdated(noteId, paragraphId, outputToUpdate);
}
logger.debug("Event from remoteproceess {}", event.getType());
} catch (Exception e) {

19
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -53,10 +53,11 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private int connectTimeout;
public RemoteInterpreterProcess(String intpRunner,
String intpDir,
Map<String, String> env,
int connectTimeout) {
this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(), connectTimeout);
String intpDir,
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener) {
this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(listener), connectTimeout);
}
RemoteInterpreterProcess(String intpRunner,
@ -260,13 +261,19 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
}
}
public void setMaxPoolSize(int size) {
if (clientPool != null) {
//Size + 2 for progress poller , cancel operation
clientPool.setMaxTotal(size + 2);
}
}
/**
* Called when angular object is updated in client side to propagate
* change to the remote process
* @param name
* @param o
*/
public void updateRemoteAngularObject(String name, String noteId, Object o) {
public void updateRemoteAngularObject(String name, String noteId, String paragraphId, Object o) {
Client client = null;
try {
client = getClient();
@ -282,7 +289,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
boolean broken = false;
try {
Gson gson = new Gson();
client.angularObjectUpdate(name, noteId, gson.toJson(o));
client.angularObjectUpdate(name, noteId, paragraphId, gson.toJson(o));
} catch (TException e) {
broken = true;
logger.error("Can't update angular object", e);

25
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java Normal file
View file

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
/**
* Event from remoteInterpreterProcess
*/
public interface RemoteInterpreterProcessListener {
public void onOutputAppend(String noteId, String paragraphId, String output);
public void onOutputUpdated(String noteId, String paragraphId, String output);
}

94
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -35,15 +35,8 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
@ -270,6 +263,7 @@ public class RemoteInterpreterServer
private Interpreter interpreter;
private String script;
private InterpreterContext context;
private Map<String, Object> infos;
public InterpretJob(
String jobId,
@ -292,7 +286,10 @@ public class RemoteInterpreterServer
@Override
public Map<String, Object> info() {
return null;
if (infos == null) {
infos = new HashMap<>();
}
return infos;
}
@Override
@ -300,7 +297,26 @@ public class RemoteInterpreterServer
try {
InterpreterContext.set(context);
InterpreterResult result = interpreter.interpret(script, context);
return result;
// data from context.out is prepended to InterpreterResult if both defined
String message = "";
context.out.flush();
InterpreterResult.Type outputType = context.out.getType();
byte[] interpreterOutput = context.out.toByteArray();
context.out.clear();
if (interpreterOutput != null && interpreterOutput.length > 0) {
message = new String(interpreterOutput);
}
String interpreterResultMessage = result.message();
if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
message += interpreterResultMessage;
return new InterpreterResult(result.code(), result.type(), message);
} else {
return new InterpreterResult(result.code(), outputType, message);
}
} finally {
InterpreterContext.remove();
}
@ -351,7 +367,8 @@ public class RemoteInterpreterServer
private InterpreterContext convert(RemoteInterpreterContext ric) {
List<InterpreterContextRunner> contextRunners = new LinkedList<InterpreterContextRunner>();
List<InterpreterContextRunner> runners = gson.fromJson(ric.getRunners(),
new TypeToken<List<RemoteInterpreterContextRunner>>(){}.getType());
new TypeToken<List<RemoteInterpreterContextRunner>>() {
}.getType());
for (InterpreterContextRunner r : runners) {
contextRunners.add(new ParagraphRunner(this, r.getNoteId(), r.getParagraphId()));
@ -366,7 +383,40 @@ public class RemoteInterpreterServer
new TypeToken<Map<String, Object>>() {}.getType()),
gson.fromJson(ric.getGui(), GUI.class),
interpreterGroup.getAngularObjectRegistry(),
contextRunners);
contextRunners, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId()));
}
private InterpreterOutput createInterpreterOutput(final String noteId, final String paragraphId) {
return new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", new String(line));
Gson gson = new Gson();
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_APPEND,
gson.toJson(appendOutput)));
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
Map<String, String> appendOutput = new HashMap<String, String>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", new String(output));
Gson gson = new Gson();
sendEvent(new RemoteInterpreterEvent(
RemoteInterpreterEventType.OUTPUT_UPDATE,
gson.toJson(appendOutput)));
}
});
}
@ -434,7 +484,7 @@ public class RemoteInterpreterServer
}
@Override
public void onRemove(String interpreterGroupId, String name, String noteId) {
public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
Map<String, String> removeObject = new HashMap<String, String>();
removeObject.put("name", name);
removeObject.put("noteId", noteId);
@ -473,15 +523,16 @@ public class RemoteInterpreterServer
* called when object is updated in client (web) side.
* @param name
* @param noteId noteId where the update issues
* @param paragraphId paragraphId where the update issues
* @param object
* @throws TException
*/
@Override
public void angularObjectUpdate(String name, String noteId, String object)
public void angularObjectUpdate(String name, String noteId, String paragraphId, String object)
throws TException {
AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry();
// first try local objects
AngularObject ao = registry.get(name, noteId);
AngularObject ao = registry.get(name, noteId, paragraphId);
if (ao == null) {
logger.error("Angular object {} not exists", name);
return;
@ -530,13 +581,13 @@ public class RemoteInterpreterServer
* Dont't need to emit event to zeppelin server
*/
@Override
public void angularObjectAdd(String name, String noteId, String object)
public void angularObjectAdd(String name, String noteId, String paragraphId, String object)
throws TException {
AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry();
// first try local objects
AngularObject ao = registry.get(name, noteId);
AngularObject ao = registry.get(name, noteId, paragraphId);
if (ao != null) {
angularObjectUpdate(name, noteId, object);
angularObjectUpdate(name, noteId, paragraphId, object);
return;
}
@ -556,12 +607,13 @@ public class RemoteInterpreterServer
value = gson.fromJson(object, String.class);
}
registry.add(name, value, noteId, false);
registry.add(name, value, noteId, paragraphId, false);
}
@Override
public void angularObjectRemove(String name, String noteId) throws TException {
public void angularObjectRemove(String name, String noteId, String paragraphId) throws
TException {
AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry();
registry.remove(name, noteId, false);
registry.remove(name, noteId, paragraphId, false);
}
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteInterpreterContext, RemoteInterpreterContext._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterContext> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext");

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterEvent implements org.apache.thrift.TBase<RemoteInterpreterEvent, RemoteInterpreterEvent._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterEvent> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent");

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java
View file

@ -33,7 +33,9 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum {
ANGULAR_OBJECT_ADD(2),
ANGULAR_OBJECT_UPDATE(3),
ANGULAR_OBJECT_REMOVE(4),
RUN_INTERPRETER_CONTEXT_RUNNER(5);
RUN_INTERPRETER_CONTEXT_RUNNER(5),
OUTPUT_APPEND(6),
OUTPUT_UPDATE(7);
private final int value;
@ -64,6 +66,10 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum {
return ANGULAR_OBJECT_REMOVE;
case 5:
return RUN_INTERPRETER_CONTEXT_RUNNER;
case 6:
return OUTPUT_APPEND;
case 7:
return OUTPUT_UPDATE;
default:
return null;
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java
View file

@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2015-8-7")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-1-4")
public class RemoteInterpreterResult implements org.apache.thrift.TBase<RemoteInterpreterResult, RemoteInterpreterResult._Fields>, java.io.Serializable, Cloneable, Comparable<RemoteInterpreterResult> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult");

425
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java
View file

File diff suppressed because it is too large Load diff

11
zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift
View file

@ -42,7 +42,9 @@ enum RemoteInterpreterEventType {
ANGULAR_OBJECT_ADD = 2,
ANGULAR_OBJECT_UPDATE = 3,
ANGULAR_OBJECT_REMOVE = 4,
RUN_INTERPRETER_CONTEXT_RUNNER = 5
RUN_INTERPRETER_CONTEXT_RUNNER = 5,
OUTPUT_APPEND = 6,
OUTPUT_UPDATE = 7
}
struct RemoteInterpreterEvent {
@ -65,7 +67,8 @@ service RemoteInterpreterService {
string getStatus(1:string jobId);
RemoteInterpreterEvent getEvent();
void angularObjectUpdate(1: string name, 2: string noteId, 3: string object);
void angularObjectAdd(1: string name, 2: string noteId, 3: string object);
void angularObjectRemove(1: string name, 2: string noteId);
void angularObjectUpdate(1: string name, 2: string noteId, 3: string paragraphId, 4: string
object);
void angularObjectAdd(1: string name, 2: string noteId, 3: string paragraphId, 4: string object);
void angularObjectRemove(1: string name, 2: string noteId, 3: string paragraphId);
}

62
zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectRegistryTest.java
View file

@ -18,6 +18,8 @@
package org.apache.zeppelin.display;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import java.util.concurrent.atomic.AtomicInteger;
@ -45,32 +47,68 @@ public class AngularObjectRegistryTest {
}
@Override
public void onRemove(String interpreterGroupId, String name, String noteId) {
public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
onRemove.incrementAndGet();
}
});
registry.add("name1", "value1", "note1");
assertEquals(1, registry.getAll("note1").size());
registry.add("name1", "value1", "note1", null);
assertEquals(1, registry.getAll("note1", null).size());
assertEquals(1, onAdd.get());
assertEquals(0, onUpdate.get());
registry.get("name1", "note1").set("newValue");
registry.get("name1", "note1", null).set("newValue");
assertEquals(1, onUpdate.get());
registry.remove("name1", "note1");
assertEquals(0, registry.getAll("note1").size());
registry.remove("name1", "note1", null);
assertEquals(0, registry.getAll("note1", null).size());
assertEquals(1, onRemove.get());
assertEquals(null, registry.get("name1", "note1"));
assertEquals(null, registry.get("name1", "note1", null));
// namespace
registry.add("name1", "value11", "note2");
assertEquals("value11", registry.get("name1", "note2").get());
assertEquals(null, registry.get("name1", "note1"));
registry.add("name1", "value11", "note2", null);
assertEquals("value11", registry.get("name1", "note2", null).get());
assertEquals(null, registry.get("name1", "note1", null));
// null namespace
registry.add("name1", "global1", null);
assertEquals("global1", registry.get("name1", null).get());
registry.add("name1", "global1", null, null);
assertEquals("global1", registry.get("name1", null, null).get());
}
@Test
public void testGetDependOnScope() {
AngularObjectRegistry registry = new AngularObjectRegistry("intpId", null);
AngularObject ao1 = registry.add("name1", "o1", "noteId1", "paragraphId1");
AngularObject ao2 = registry.add("name2", "o2", "noteId1", "paragraphId1");
AngularObject ao3 = registry.add("name2", "o3", "noteId1", "paragraphId2");
AngularObject ao4 = registry.add("name3", "o4", "noteId1", null);
AngularObject ao5 = registry.add("name4", "o5", null, null);
assertNull(registry.get("name3", "noteId1", "paragraphId1"));
assertNull(registry.get("name1", "noteId2", null));
assertEquals("o1", registry.get("name1", "noteId1", "paragraphId1").get());
assertEquals("o2", registry.get("name2", "noteId1", "paragraphId1").get());
assertEquals("o3", registry.get("name2", "noteId1", "paragraphId2").get());
assertEquals("o4", registry.get("name3", "noteId1", null).get());
assertEquals("o5", registry.get("name4", null, null).get());
}
@Test
public void testGetAllDependOnScope() {
AngularObjectRegistry registry = new AngularObjectRegistry("intpId", null);
AngularObject ao1 = registry.add("name1", "o", "noteId1", "paragraphId1");
AngularObject ao2 = registry.add("name2", "o", "noteId1", "paragraphId1");
AngularObject ao3 = registry.add("name2", "o", "noteId1", "paragraphId2");
AngularObject ao4 = registry.add("name3", "o", "noteId1", null);
AngularObject ao5 = registry.add("name4", "o", null, null);
assertEquals(2, registry.getAll("noteId1", "paragraphId1").size());
assertEquals(1, registry.getAll("noteId1", "paragraphId2").size());
assertEquals(1, registry.getAll("noteId1", null).size());
assertEquals(1, registry.getAll(null, null).size());
assertEquals(5, registry.getAllWithGlobal("noteId1").size());
}
}

55
zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/AngularObjectTest.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.display;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
import java.util.concurrent.atomic.AtomicInteger;
@ -26,10 +27,60 @@ import org.junit.Test;
public class AngularObjectTest {
@Test
public void testEquals() {
assertEquals(
new AngularObject("name", "value", "note1", null, null),
new AngularObject("name", "value", "note1", null, null)
);
assertEquals(
new AngularObject("name", "value", "note1", "paragraph1", null),
new AngularObject("name", "value", "note1", "paragraph1", null)
);
assertEquals(
new AngularObject("name", "value", null, null, null),
new AngularObject("name", "value", null, null, null)
);
assertEquals(
new AngularObject("name", "value1", null, null, null),
new AngularObject("name", "value2", null, null, null)
);
assertNotSame(
new AngularObject("name1", "value", null, null, null),
new AngularObject("name2", "value", null, null, null)
);
assertNotSame(
new AngularObject("name1", "value", "note1", null, null),
new AngularObject("name2", "value", "note2", null, null)
);
assertNotSame(
new AngularObject("name1", "value", "note", null, null),
new AngularObject("name2", "value", null, null, null)
);
assertNotSame(
new AngularObject("name", "value", "note", "paragraph1", null),
new AngularObject("name", "value", "note", "paragraph2", null)
);
assertNotSame(
new AngularObject("name", "value", "note1", null, null),
new AngularObject("name", "value", "note1", "paragraph1", null)
);
}
@Test
public void testListener() {
final AtomicInteger updated = new AtomicInteger(0);
AngularObject ao = new AngularObject("name", "value", "note1", new AngularObjectListener() {
AngularObject ao = new AngularObject("name", "value", "note1", null, new AngularObjectListener() {
@Override
public void updated(AngularObject updatedObject) {
@ -55,7 +106,7 @@ public class AngularObjectTest {
public void testWatcher() throws InterruptedException {
final AtomicInteger updated = new AtomicInteger(0);
final AtomicInteger onWatch = new AtomicInteger(0);
AngularObject ao = new AngularObject("name", "value", "note1", new AngularObjectListener() {
AngularObject ao = new AngularObject("name", "value", "note1", null, new AngularObjectListener() {
@Override
public void updated(AngularObject updatedObject) {
updated.incrementAndGet();

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java
View file

@ -27,7 +27,7 @@ public class InterpreterContextTest {
public void testThreadLocal() {
assertNull(InterpreterContext.get());
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null));
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null));
assertNotNull(InterpreterContext.get());
InterpreterContext.remove();

Some files were not shown because too many files have changed in this diff Show more